A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: shared colors and icons with transforms
dextmorgn6dcc1e0f0248
diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.pyindex 7026a893..f17971f7 100644--- a/flowsint-api/app/api/routes/transforms.py+++ b/flowsint-api/app/api/routes/transforms.py@@ -19,8 +19,9 @@ from app.api.deps import get_current_userfrom app.api.schemas.transform import TransformRead, TransformCreate, TransformUpdatefrom app.types.asn import ASNfrom app.types.cidr import CIDR-from app.types.wallet import Wallet, WalletTransaction+from app.types.wallet import CryptoWallet, CryptoWalletTransaction, CryptoNFTfrom app.types.website import Website+from app.types.individual import Individualclass FlowComputationRequest(BaseModel):nodes: List[Node]@@ -62,12 +63,13 @@ def get_transforms(# Returns the "raw_materials" for the transform editor@router.get("/raw_materials")-async def get_scans_list(current_user: Profile = Depends(get_current_user)):+async def get_scans_list():scanners = ScannerRegistry.list_by_category()flattened_scanners = {category: [{"class_name": scanner["class_name"],+ "category": scanner["category"],"name": scanner["name"],"module": scanner["module"],"doc": scanner["doc"],@@ -83,6 +85,7 @@ async def get_scans_list(current_user: Profile = Depends(get_current_user)):# Ajoute les types comme des "scanners" spéciaux de type 'type'object_inputs = [extract_input_schema("Organization", Organization),+ extract_input_schema("Individual", Individual),extract_input_schema("Domain", Domain),extract_input_schema("Website", Website),extract_input_schema("Ip", Ip),@@ -90,9 +93,9 @@ async def get_scans_list(current_user: Profile = Depends(get_current_user)):extract_input_schema("CIDR", CIDR),extract_input_schema("SocialProfile", SocialProfile),extract_input_schema("Email", Email),- extract_input_schema("CryptoWallet", Wallet),- extract_input_schema("CryptoTransaction", WalletTransaction),-+ extract_input_schema("CryptoWallet", CryptoWallet),+ extract_input_schema("CryptoTransaction", CryptoWalletTransaction),+ extract_input_schema("CryptoNFT", CryptoNFT)]flattened_scanners["types"] = object_inputsdiff --git a/flowsint-api/app/scanners/crypto/wallet_to_nfts.py b/flowsint-api/app/scanners/crypto/wallet_to_nfts.pyindex 8f7cd43f..229a44c7 100644--- a/flowsint-api/app/scanners/crypto/wallet_to_nfts.py+++ b/flowsint-api/app/scanners/crypto/wallet_to_nfts.py@@ -4,11 +4,11 @@ from typing import List, Dict, Any, TypeAlias, Unionfrom pydantic import TypeAdapterimport requestsfrom app.scanners.base import Scanner-from app.types.wallet import Wallet, NFT+from app.types.wallet import CryptoWallet, CryptoNFTfrom app.utils import resolve_typefrom app.core.logger import Logger-InputType: TypeAlias = List[Wallet]-OutputType: TypeAlias = List[NFT]+InputType: TypeAlias = List[CryptoWallet]+OutputType: TypeAlias = List[CryptoNFT]ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")@@ -22,7 +22,7 @@ class WalletAddressToNFTs(Scanner):@classmethoddef category(cls) -> str:- return "crypto"+ return "CryptoWallet"@classmethoddef key(cls) -> str:@@ -59,10 +59,10 @@ class WalletAddressToNFTs(Scanner):for item in data:wallet_obj = Noneif isinstance(item, str):- wallet_obj = Wallet(address=item)+ wallet_obj = CryptoWallet(address=item)elif isinstance(item, dict) and "address" in item:- wallet_obj = Wallet(address=item["address"])- elif isinstance(item, Wallet):+ wallet_obj = CryptoWallet(address=item["address"])+ elif isinstance(item, CryptoWallet):wallet_obj = itemif wallet_obj:cleaned.append(wallet_obj)@@ -78,7 +78,7 @@ class WalletAddressToNFTs(Scanner):print(f"Error resolving nfts for {d.address}: {e}")return results- def _get_nfts(self, address: str) -> List[NFT]:+ def _get_nfts(self, address: str) -> List[CryptoNFT]:nfts = []"""Get nfts for a wallet address."""params = {@@ -96,8 +96,8 @@ class WalletAddressToNFTs(Scanner):data = response.json()results = data["result"]for tx in results:- nfts.append(NFT(- wallet=Wallet(address=address),+ nfts.append(CryptoNFT(+ wallet=CryptoWallet(address=address),contract_address=tx["contractAddress"],token_id=tx["tokenID"],collection_name=tx["collectionName"],diff --git a/flowsint-api/app/scanners/crypto/wallet_to_transactions.py b/flowsint-api/app/scanners/crypto/wallet_to_transactions.pyindex c160e581..4eb74489 100644--- a/flowsint-api/app/scanners/crypto/wallet_to_transactions.py+++ b/flowsint-api/app/scanners/crypto/wallet_to_transactions.py@@ -4,11 +4,11 @@ from pydantic import TypeAdapterimport requestsfrom datetime import datetimefrom app.scanners.base import Scanner-from app.types.wallet import Wallet, WalletTransaction+from app.types.wallet import CryptoWallet, CryptoWalletTransactionfrom app.utils import resolve_typefrom app.core.logger import Logger-InputType: TypeAlias = List[Wallet]-OutputType: TypeAlias = List[WalletTransaction]+InputType: TypeAlias = List[CryptoWallet]+OutputType: TypeAlias = List[CryptoWalletTransaction]ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")@@ -16,7 +16,7 @@ ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")def wei_to_eth(wei_str):return int(wei_str) / 10**18-class WalletAddressToTransactions(Scanner):+class CryptoWalletAddressToTransactions(Scanner):"""Resolve transactions for a wallet address (ETH)."""@classmethod@@ -25,7 +25,7 @@ class WalletAddressToTransactions(Scanner):@classmethoddef category(cls) -> str:- return "crypto"+ return "CryptoCryptoWallet"@classmethoddef key(cls) -> str:@@ -62,10 +62,10 @@ class WalletAddressToTransactions(Scanner):for item in data:wallet_obj = Noneif isinstance(item, str):- wallet_obj = Wallet(address=item)+ wallet_obj = CryptoWallet(address=item)elif isinstance(item, dict) and "address" in item:- wallet_obj = Wallet(address=item["address"])- elif isinstance(item, Wallet):+ wallet_obj = CryptoWallet(address=item["address"])+ elif isinstance(item, CryptoWallet):wallet_obj = itemif wallet_obj:cleaned.append(wallet_obj)@@ -81,7 +81,7 @@ class WalletAddressToTransactions(Scanner):Logger.error(self.sketch_id, {"message": f"Error resolving transactions for {d.address}: {e}"})return results- def _get_transactions(self, address: str) -> List[WalletTransaction]:+ def _get_transactions(self, address: str) -> List[CryptoWalletTransaction]:transactions = []"""Get transactions for a wallet address."""params = {@@ -100,11 +100,11 @@ class WalletAddressToTransactions(Scanner):results = data["result"]for tx in results:if tx["to"] !=None:- target = Wallet(address=tx["to"])+ target = CryptoWallet(address=tx["to"])else:- target = Wallet(address=tx["contractAddress"])- transactions.append(WalletTransaction(- source=Wallet(address=address),+ target = CryptoWallet(address=tx["contractAddress"])+ transactions.append(CryptoWalletTransaction(+ source=CryptoWallet(address=address),target=target,hash=tx["hash"],value=wei_to_eth(tx["value"]),diff --git a/flowsint-api/app/scanners/domains/resolve.py b/flowsint-api/app/scanners/domains/resolve.pyindex 7554ffdb..113548e3 100644--- a/flowsint-api/app/scanners/domains/resolve.py+++ b/flowsint-api/app/scanners/domains/resolve.py@@ -4,7 +4,7 @@ from pydantic import TypeAdapterfrom app.scanners.base import Scannerfrom app.types.domain import Domainfrom app.types.ip import Ip-from app.utils import is_valid_domain, resolve_type+from app.utils import is_valid_domain, resolve_type, is_root_domainimport uuidfrom app.types.transform import Node, Edgefrom app.core.logger import Logger@@ -58,11 +58,12 @@ class ResolveScanner(Scanner):for item in data:domain_obj = Noneif isinstance(item, str):- domain_obj = Domain(domain=item)+ domain_obj = Domain(domain=item, root=is_root_domain(item))elif isinstance(item, dict) and "domain" in item:- domain_obj = Domain(domain=item["domain"])+ domain_obj = Domain(domain=item["domain"], root=is_root_domain(item["domain"]))elif isinstance(item, Domain):- domain_obj = item+ # If the Domain object already exists, update its root field+ domain_obj = Domain(domain=item.domain, root=is_root_domain(item.domain))if domain_obj and is_valid_domain(domain_obj.domain):cleaned.append(domain_obj)return cleaned@@ -83,7 +84,7 @@ class ResolveScanner(Scanner):MERGE (d:domain {domain: $domain})SET d.sketch_id = $sketch_id,d.label = $domain,- d.type = "domain"+ d.type = $typeMERGE (ip:ip {address: $ip})SET ip.sketch_id = $sketch_id,ip.label = $label,@@ -96,6 +97,7 @@ class ResolveScanner(Scanner):"ip": ip_obj.address,"sketch_id": self.sketch_id,"label": ip_obj.address,+ "type": "domain" if domain_obj.root else "subdomain"})nodes = [Node(id=str(uuid.uuid4()),diff --git a/flowsint-api/app/scanners/domains/subdomains.py b/flowsint-api/app/scanners/domains/subdomains.pyindex 029690b3..45e65010 100644--- a/flowsint-api/app/scanners/domains/subdomains.py+++ b/flowsint-api/app/scanners/domains/subdomains.py@@ -7,7 +7,7 @@ from app.types.domain import Domain, Domainfrom app.utils import is_valid_domain, resolve_typefrom pydantic import TypeAdapterfrom app.core.logger import Logger-+from app.tools.network.subfinder import SubfinderToolInputType: TypeAlias = List[Domain]OutputType: TypeAlias = List[Domain]@@ -55,9 +55,7 @@ class SubdomainScanner(Scanner):def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:cleaned: InputType = []- invalid_items = []-- for i, item in enumerate(data):+ for item in data:domain_obj = Noneif isinstance(item, str):domain_obj = Domain(domain=item)@@ -65,21 +63,10 @@ class SubdomainScanner(Scanner):domain_obj = Domain(domain=item["domain"])elif isinstance(item, Domain):domain_obj = item- else:- invalid_items.append(f"Item at index {i}: {type(item).__name__} - {item}")- continue-if domain_obj and is_valid_domain(domain_obj.domain):cleaned.append(domain_obj)- else:- invalid_items.append(f"Item at index {i}: Invalid domain '{getattr(domain_obj, 'domain', item) if domain_obj else item}'")-- if invalid_items:- error_msg = f"Invalid input format. The following items could not be processed:\n" + "\n".join(invalid_items)- Logger.error(self.sketch_id, {"message":error_msg})- raise ValueError(error_msg)-return cleaned+def scan(self, data: InputType) -> OutputType:"""Find subdomains using subfinder (if available) or fallback to crt.sh."""@@ -127,18 +114,9 @@ class SubdomainScanner(Scanner):def __get_subdomains_from_subfinder(self, domain: str) -> set[str]:subdomains: set[str] = set()+ subfinder = SubfinderTool()try:- result = subprocess.run(- ["subfinder", "-silent", "-d", domain],- capture_output=True, text=True, timeout=60- )- if result.returncode == 0:- for sub in result.stdout.strip().splitlines():- sub = sub.strip().lower()- if is_valid_domain(sub) and sub.endswith(domain) and sub != domain and not sub.startswith("."):- subdomains.add(sub)- else:- Logger.error(self.sketch_id, f"subfinder failed for {domain}: {result.stderr.strip()}")+ subdomains = subfinder.launch(domain)except Exception as e:Logger.error(self.sketch_id, f"subfinder exception for {domain}: {e}")return subdomains@@ -163,7 +141,7 @@ class SubdomainScanner(Scanner):"subdomain": subdomain,"sketch_id": self.sketch_id,"label": subdomain,- "type": "domain"+ "type": "subdomain"})Logger.graph_append(self.sketch_id, {"message":f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found."})diff --git a/flowsint-api/app/scanners/individuals/to_org.py b/flowsint-api/app/scanners/individuals/to_org.pynew file mode 100644index 00000000..c65259f1--- /dev/null+++ b/flowsint-api/app/scanners/individuals/to_org.py@@ -0,0 +1,510 @@+import requests+from typing import List, Dict, Any, TypeAlias, Union+from pydantic import TypeAdapter+from app.scanners.base import Scanner+from app.types.organization import Organization+from app.types.individual import Individual+from app.utils import resolve_type+from app.core.logger import Logger+from app.tools.organizations.sirene import SireneTool+++InputType: TypeAlias = List[Individual]+OutputType: TypeAlias = List[Organization]++class IndividualToOrgScanner(Scanner):+ """Find organization from a person with data from SIRENE (France only)."""++ @classmethod+ def name(cls) -> str:+ return "to_org"++ @classmethod+ def category(cls) -> str:+ return "Individual"++ @classmethod+ def key(cls) -> str:+ return "fullname"++ @classmethod+ def input_schema(cls) -> Dict[str, Any]:+ adapter = TypeAdapter(InputType)+ schema = adapter.json_schema()+ # Find the Organization type in $defs+ organization_def = schema["$defs"].get("Individual")+ if not organization_def:+ raise ValueError("Individual type not found in schema")+ return {+ "type": "Individual",+ "properties": [+ {"name": "fullname", "type": "string"}+ ]+ }++ @classmethod+ def output_schema(cls) -> Dict[str, Any]:+ adapter = TypeAdapter(OutputType)+ schema = adapter.json_schema()+ # the items property contains the Organization type reference+ items_schema = schema.get("items", {})+ if "$ref" in items_schema:+ # Extract the type name from the $ref (e.g., "#/$defs/Organization" -> "Organization")+ ref_path = items_schema["$ref"]+ type_name = ref_path.split("/")[-1]+ organization_def = schema["$defs"].get(type_name)+ if not organization_def:+ raise ValueError(f"Type {type_name} not found in schema")+ return {+ "type": type_name,+ "properties": [+ {"name": prop, "type": resolve_type(info, schema)}+ for prop, info in organization_def["properties"].items()+ ]+ }+ else:+ raise ValueError("Expected $ref in items schema for List type")+++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ if not isinstance(data, list):+ raise ValueError(f"Expected list input, got {type(data).__name__}")+ cleaned: InputType = []+ for item in data:+ if isinstance(item, str):+ parts = item.strip().split()+ if len(parts) >= 2:+ firstname = parts[0]+ lastname = " ".join(parts[1:])+ cleaned.append(Individual(first_name=firstname, last_name=lastname, full_name=item))+ elif isinstance(item, dict) and item.get("first_name") and item.get("last_name"):+ cleaned.append(Individual(**item))+ elif isinstance(item, Individual):+ cleaned.append(item)+ if(len(cleaned)==0):+ Logger.error(self.sketch_id, {"message": "[INDIVIDUAL_TO_ORG] The input type did not match."})+ return cleaned++ def scan(self, data: InputType) -> OutputType:++ results: OutputType = []+ for indiv in data:+ try:+ sirene = SireneTool()+ raw_orgs = sirene.launch(f'{indiv.first_name}+{indiv.last_name}', limit=25)+ if len(raw_orgs)> 0:+ for org_dict in raw_orgs:+ enriched_org = self.enrich_org(org_dict)+ if enriched_org is not None:+ results.append(enriched_org)+ except Exception as e:+ continue+ return results++ def enrich_org(self, company: Dict) -> Organization:+ try:+ # Extract siege data+ siege = company.get("siege", {})+ # Create PhysicalAddress for siege_geo_adresse if coordinates exist+ siege_geo_adresse = None+ if siege.get("latitude") and siege.get("longitude"):+ from app.types.address import PhysicalAddress+ siege_geo_adresse = PhysicalAddress(+ address=siege.get("adresse", ""),+ city=siege.get("libelle_commune", ""),+ country="FR", # SIRENE is French registry+ zip=siege.get("code_postal", ""),+ latitude=float(siege.get("latitude", 0)),+ longitude=float(siege.get("longitude", 0))+ )++ # Extract dirigeants and convert to Individual objects+ dirigeants = []+ for dirigeant_data in company.get("dirigeants", []):+ from app.types.individual import Individual+ dirigeant = Individual(+ first_name=dirigeant_data.get("prenoms", ""),+ last_name=dirigeant_data.get("nom", ""),+ full_name=f"{dirigeant_data.get('prenoms', '')} {dirigeant_data.get('nom', '')}".strip(),+ birth_date=dirigeant_data.get("date_de_naissance"),+ gender=None # Not available in SIRENE data+ )+ dirigeants.append(dirigeant)++ name = company.get("nom_raison_sociale") or company.get("nom_complet")++ # Ensure we have a valid name+ if not name:+ Logger.error(self.sketch_id, {"message": f"Organization has no valid name: {company}"})+ return None++ return Organization(+ # Basic information+ name=name,+ siren=company.get("siren"),+ nom_complet=company.get("nom_complet"),+ nom_raison_sociale=company.get("nom_raison_sociale"),+ sigle=company.get("sigle"),++ # Establishment information+ nombre_etablissements=company.get("nombre_etablissements"),+ nombre_etablissements_ouverts=company.get("nombre_etablissements_ouverts"),++ # Activity information+ activite_principale=company.get("activite_principale"),+ section_activite_principale=company.get("section_activite_principale"),++ # Company category+ categorie_entreprise=company.get("categorie_entreprise"),+ annee_categorie_entreprise=company.get("annee_categorie_entreprise"),++ # Employment information+ caractere_employeur=company.get("caractere_employeur"),+ tranche_effectif_salarie=company.get("tranche_effectif_salarie"),+ annee_tranche_effectif_salarie=company.get("annee_tranche_effectif_salarie"),++ # Dates+ date_creation=company.get("date_creation"),+ date_fermeture=company.get("date_fermeture"),+ date_mise_a_jour=company.get("date_mise_a_jour"),+ date_mise_a_jour_insee=company.get("date_mise_a_jour_insee"),+ date_mise_a_jour_rne=company.get("date_mise_a_jour_rne"),++ # Legal information+ nature_juridique=company.get("nature_juridique"),+ etat_administratif=company.get("etat_administratif"),+ statut_diffusion=company.get("statut_diffusion"),++ # Siege (Headquarters) information+ siege_activite_principale=siege.get("activite_principale"),+ siege_activite_principale_registre_metier=siege.get("activite_principale_registre_metier"),+ siege_annee_tranche_effectif_salarie=siege.get("annee_tranche_effectif_salarie"),+ siege_adresse=siege.get("adresse"),+ siege_caractere_employeur=siege.get("caractere_employeur"),+ siege_cedex=siege.get("cedex"),+ siege_code_pays_etranger=siege.get("code_pays_etranger"),+ siege_code_postal=siege.get("code_postal"),+ siege_commune=siege.get("commune"),+ siege_complement_adresse=siege.get("complement_adresse"),+ siege_coordonnees=siege.get("coordonnees"),+ siege_date_creation=siege.get("date_creation"),+ siege_date_debut_activite=siege.get("date_debut_activite"),+ siege_date_fermeture=siege.get("date_fermeture"),+ siege_date_mise_a_jour=siege.get("date_mise_a_jour"),+ siege_date_mise_a_jour_insee=siege.get("date_mise_a_jour_insee"),+ siege_departement=siege.get("departement"),+ siege_distribution_speciale=siege.get("distribution_speciale"),+ siege_epci=siege.get("epci"),+ siege_est_siege=siege.get("est_siege"),+ siege_etat_administratif=siege.get("etat_administratif"),+ siege_geo_adresse=siege_geo_adresse,+ siege_geo_id=siege.get("geo_id"),+ siege_indice_repetition=siege.get("indice_repetition"),+ siege_latitude=siege.get("latitude"),+ siege_libelle_cedex=siege.get("libelle_cedex"),+ siege_libelle_commune=siege.get("libelle_commune"),+ siege_libelle_commune_etranger=siege.get("libelle_commune_etranger"),+ siege_libelle_pays_etranger=siege.get("libelle_pays_etranger"),+ siege_libelle_voie=siege.get("libelle_voie"),+ siege_liste_enseignes=siege.get("liste_enseignes"),+ siege_liste_finess=siege.get("liste_finess"),+ siege_liste_id_bio=siege.get("liste_id_bio"),+ siege_liste_idcc=siege.get("liste_idcc"),+ siege_liste_id_organisme_formation=siege.get("liste_id_organisme_formation"),+ siege_liste_rge=siege.get("liste_rge"),+ siege_liste_uai=siege.get("liste_uai"),+ siege_longitude=siege.get("longitude"),+ siege_nom_commercial=siege.get("nom_commercial"),+ siege_numero_voie=siege.get("numero_voie"),+ siege_region=siege.get("region"),+ siege_siret=siege.get("siret"),+ siege_statut_diffusion_etablissement=siege.get("statut_diffusion_etablissement"),+ siege_tranche_effectif_salarie=siege.get("tranche_effectif_salarie"),+ siege_type_voie=siege.get("type_voie"),++ # Dirigeants (Leaders)+ dirigeants=dirigeants if dirigeants else None,++ # Matching establishments+ matching_etablissements=company.get("matching_etablissements"),++ # Financial information+ finances=company.get("finances"),++ # Complements (Additional information)+ complements_collectivite_territoriale=company.get("complements", {}).get("collectivite_territoriale"),+ complements_convention_collective_renseignee=company.get("complements", {}).get("convention_collective_renseignee"),+ complements_liste_idcc=company.get("complements", {}).get("liste_idcc"),+ complements_egapro_renseignee=company.get("complements", {}).get("egapro_renseignee"),+ complements_est_achats_responsables=company.get("complements", {}).get("est_achats_responsables"),+ complements_est_alim_confiance=company.get("complements", {}).get("est_alim_confiance"),+ complements_est_association=company.get("complements", {}).get("est_association"),+ complements_est_bio=company.get("complements", {}).get("est_bio"),+ complements_est_entrepreneur_individuel=company.get("complements", {}).get("est_entrepreneur_individuel"),+ complements_est_entrepreneur_spectacle=company.get("complements", {}).get("est_entrepreneur_spectacle"),+ complements_est_ess=company.get("complements", {}).get("est_ess"),+ complements_est_finess=company.get("complements", {}).get("est_finess"),+ complements_est_organisme_formation=company.get("complements", {}).get("est_organisme_formation"),+ complements_est_qualiopi=company.get("complements", {}).get("est_qualiopi"),+ complements_liste_id_organisme_formation=company.get("complements", {}).get("liste_id_organisme_formation"),+ complements_est_rge=company.get("complements", {}).get("est_rge"),+ complements_est_service_public=company.get("complements", {}).get("est_service_public"),+ complements_est_l100_3=company.get("complements", {}).get("est_l100_3"),+ complements_est_siae=company.get("complements", {}).get("est_siae"),+ complements_est_societe_mission=company.get("complements", {}).get("est_societe_mission"),+ complements_est_uai=company.get("complements", {}).get("est_uai"),+ complements_est_patrimoine_vivant=company.get("complements", {}).get("est_patrimoine_vivant"),+ complements_bilan_ges_renseigne=company.get("complements", {}).get("bilan_ges_renseigne"),+ complements_identifiant_association=company.get("complements", {}).get("identifiant_association"),+ complements_statut_entrepreneur_spectacle=company.get("complements", {}).get("statut_entrepreneur_spectacle"),+ complements_type_siae=company.get("complements", {}).get("type_siae")+ )+ except Exception as e:+ Logger.error(self.sketch_id, {"message": f"Error enriching organization: {e}"})+ return None++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ if not self.neo4j_conn:+ return results++ for org in results:+ # Create or update the organization node with all SIRENE properties+ self.neo4j_conn.query("""+ MERGE (o:Organization {name: $name, country: $country})+ SET o.siren = $siren,+ o.nom_complet = $nom_complet,+ o.nom_raison_sociale = $nom_raison_sociale,+ o.sigle = $sigle,+ o.nombre_etablissements = $nombre_etablissements,+ o.nombre_etablissements_ouverts = $nombre_etablissements_ouverts,+ o.activite_principale = $activite_principale,+ o.section_activite_principale = $section_activite_principale,+ o.categorie_entreprise = $categorie_entreprise,+ o.annee_categorie_entreprise = $annee_categorie_entreprise,+ o.caractere_employeur = $caractere_employeur,+ o.tranche_effectif_salarie = $tranche_effectif_salarie,+ o.annee_tranche_effectif_salarie = $annee_tranche_effectif_salarie,+ o.date_creation = $date_creation,+ o.date_fermeture = $date_fermeture,+ o.date_mise_a_jour = $date_mise_a_jour,+ o.date_mise_a_jour_insee = $date_mise_a_jour_insee,+ o.date_mise_a_jour_rne = $date_mise_a_jour_rne,+ o.nature_juridique = $nature_juridique,+ o.etat_administratif = $etat_administratif,+ o.statut_diffusion = $statut_diffusion,+ o.sketch_id = $sketch_id,+ o.label = $name,+ o.caption = $name,+ o.type = 'organization'+ """, {+ "name": org.name,+ "country": "FR",+ "siren": org.siren,+ "nom_complet": org.nom_complet,+ "nom_raison_sociale": org.nom_raison_sociale,+ "sigle": org.sigle,+ "nombre_etablissements": org.nombre_etablissements,+ "nombre_etablissements_ouverts": org.nombre_etablissements_ouverts,+ "activite_principale": org.activite_principale,+ "section_activite_principale": org.section_activite_principale,+ "categorie_entreprise": org.categorie_entreprise,+ "annee_categorie_entreprise": org.annee_categorie_entreprise,+ "caractere_employeur": org.caractere_employeur,+ "tranche_effectif_salarie": org.tranche_effectif_salarie,+ "annee_tranche_effectif_salarie": org.annee_tranche_effectif_salarie,+ "date_creation": org.date_creation,+ "date_fermeture": org.date_fermeture,+ "date_mise_a_jour": org.date_mise_a_jour,+ "date_mise_a_jour_insee": org.date_mise_a_jour_insee,+ "date_mise_a_jour_rne": org.date_mise_a_jour_rne,+ "nature_juridique": org.nature_juridique,+ "etat_administratif": org.etat_administratif,+ "statut_diffusion": org.statut_diffusion,+ "sketch_id": self.sketch_id,+ })++ # Add SIREN as identifier if available+ if org.siren:+ self.neo4j_conn.query("""+ MERGE (i:Identifier {type: 'siren', value: $value})+ SET i.country = $country,+ i.issued_by = $issued_by,+ i.sketch_id = $sketch_id,+ i.label = $label,+ i.caption = $label+ WITH i+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_IDENTIFIER {sketch_id: $sketch_id}]->(i)+ """, {+ "type": "siren",+ "label": f"SIREN: {org.siren}",+ "caption": f"SIREN: {org.siren}",+ "value": org.siren,+ "country": "FR",+ "issued_by": "INSEE",+ "sketch_id": self.sketch_id,+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: SIREN {org.siren} -> {org.name}"})++ # Add SIRET as identifier if available+ if org.siege_siret:+ self.neo4j_conn.query("""+ MERGE (i:Identifier {type: 'siret', value: $value})+ SET i.country = $country,+ i.issued_by = $issued_by,+ i.sketch_id = $sketch_id,+ i.label = $label,+ i.caption = $label+ WITH i+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_IDENTIFIER {sketch_id: $sketch_id}]->(i)+ """, {+ "type": "siret",+ "label": f"SIRET: {org.siege_siret}",+ "caption": f"SIRET: {org.siege_siret}",+ "value": org.siege_siret,+ "country": "FR",+ "issued_by": "INSEE",+ "sketch_id": self.sketch_id,+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: SIRET {org.siege_siret} -> {org.name}"})++ # Add dirigeants (leaders) as Individual nodes with relationships+ if org.dirigeants:+ for dirigeant in org.dirigeants:+ self.neo4j_conn.query("""+ MERGE (i:Individual {full_name: $full_name})+ SET i.first_name = $first_name,+ i.last_name = $last_name,+ i.birth_date = $birth_date,+ i.gender = $gender,+ i.sketch_id = $sketch_id,+ i.label = $full_name,+ i.caption = $full_name,+ i.type = 'individual'+ WITH i+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_LEADER {sketch_id: $sketch_id}]->(i)+ """, {+ "full_name": dirigeant.full_name,+ "first_name": dirigeant.first_name,+ "last_name": dirigeant.last_name,+ "birth_date": dirigeant.birth_date,+ "gender": dirigeant.gender,+ "sketch_id": self.sketch_id,+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_LEADER -> {dirigeant.full_name}"})++ # Add siege address as PhysicalAddress node if available+ if org.siege_geo_adresse:+ address = org.siege_geo_adresse+ self.neo4j_conn.query("""+ MERGE (a:PhysicalAddress {address: $address, city: $city, country: $country})+ SET a.zip = $zip,+ a.latitude = $latitude,+ a.longitude = $longitude,+ a.sketch_id = $sketch_id,+ a.label = $label,+ a.caption = $caption,+ a.type = 'location'+ WITH a+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_ADDRESS {sketch_id: $sketch_id}]->(a)+ """, {+ "address": address.address,+ "city": address.city,+ "country": address.country,+ "zip": address.zip,+ "latitude": address.latitude,+ "longitude": address.longitude,+ "sketch_id": self.sketch_id,+ "label": f"{address.address}, {address.city}",+ "caption": f"{address.address}, {address.city}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}"})++ # Add siege location as Location node if coordinates are available but no PhysicalAddress+ elif org.siege_latitude and org.siege_longitude:+ self.neo4j_conn.query("""+ MERGE (l:Location {latitude: $latitude, longitude: $longitude})+ SET l.address = $address,+ l.city = $city,+ l.country = $country,+ l.zip = $zip,+ l.sketch_id = $sketch_id,+ l.label = $label,+ l.caption = $caption,+ l.type = 'location'+ WITH l+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:LOCATED_AT {sketch_id: $sketch_id}]->(l)+ """, {+ "latitude": float(org.siege_latitude),+ "longitude": float(org.siege_longitude),+ "address": org.siege_adresse,+ "city": org.siege_libelle_commune,+ "country": "FR",+ "zip": org.siege_code_postal,+ "sketch_id": self.sketch_id,+ "label": f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",+ "caption": f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}"})++ # Add activity codes as Activity nodes+ if org.activite_principale:+ self.neo4j_conn.query("""+ MERGE (a:Activity {code: $code})+ SET a.section = $section,+ a.sketch_id = $sketch_id,+ a.label = $label,+ a.caption = $caption,+ a.type = 'activity'+ WITH a+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_ACTIVITY {sketch_id: $sketch_id}]->(a)+ """, {+ "code": org.activite_principale,+ "section": org.section_activite_principale,+ "sketch_id": self.sketch_id,+ "label": f"Activity: {org.activite_principale}",+ "caption": f"Activity: {org.activite_principale}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}"})++ # Add legal nature as LegalNature node+ if org.nature_juridique:+ self.neo4j_conn.query("""+ MERGE (l:LegalNature {code: $code})+ SET l.sketch_id = $sketch_id,+ l.label = $label,+ l.caption = $caption,+ l.type = 'legal_nature'+ WITH l+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_LEGAL_NATURE {sketch_id: $sketch_id}]->(l)+ """, {+ "code": org.nature_juridique,+ "sketch_id": self.sketch_id,+ "label": f"Legal Nature: {org.nature_juridique}",+ "caption": f"Legal Nature: {org.nature_juridique}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}"})++ return results++diff --git a/flowsint-api/app/scanners/ips/asn_to_cidrs.py b/flowsint-api/app/scanners/ips/asn_to_cidrs.pyindex 49dcdafd..14d51844 100644--- a/flowsint-api/app/scanners/ips/asn_to_cidrs.py+++ b/flowsint-api/app/scanners/ips/asn_to_cidrs.py@@ -22,7 +22,7 @@ class AsnToCidrsScanner(Scanner):@classmethoddef category(cls) -> str:- return "asns"+ return "Asn"@classmethoddef input_schema(cls) -> Dict[str, Any]:diff --git a/flowsint-api/app/scanners/ips/cidr_to_ips.py b/flowsint-api/app/scanners/ips/cidr_to_ips.pyindex 40bc4d15..8fa118df 100644--- a/flowsint-api/app/scanners/ips/cidr_to_ips.py+++ b/flowsint-api/app/scanners/ips/cidr_to_ips.py@@ -19,7 +19,7 @@ class CidrToIpsScanner(Scanner):@classmethoddef category(cls) -> str:- return "cidrs"+ return "Cidr"@classmethoddef input_schema(cls) -> Dict[str, Any]:diff --git a/flowsint-api/app/scanners/ips/geolocation.py b/flowsint-api/app/scanners/ips/geolocation.pyindex 837541b3..468a60e2 100644--- a/flowsint-api/app/scanners/ips/geolocation.py+++ b/flowsint-api/app/scanners/ips/geolocation.py@@ -18,7 +18,7 @@ class GeolocationScanner(Scanner):@classmethoddef category(cls) -> str:- return "ips"+ return "Ip"@classmethoddef input_schema(cls) -> Dict[str, Any]:diff --git a/flowsint-api/app/scanners/ips/ip_to_asn.py b/flowsint-api/app/scanners/ips/ip_to_asn.pyindex 4a5661bb..03e3e402 100644--- a/flowsint-api/app/scanners/ips/ip_to_asn.py+++ b/flowsint-api/app/scanners/ips/ip_to_asn.py@@ -22,7 +22,7 @@ class IpToAsnScanner(Scanner):@classmethoddef category(cls) -> str:- return "ips"+ return "Ip"@classmethoddef key(cls) -> str:diff --git a/flowsint-api/app/scanners/ips/reverse_resolve.py b/flowsint-api/app/scanners/ips/reverse_resolve.pyindex f0d53dad..42459173 100644--- a/flowsint-api/app/scanners/ips/reverse_resolve.py+++ b/flowsint-api/app/scanners/ips/reverse_resolve.py@@ -25,7 +25,7 @@ class ReverseResolveScanner(Scanner):@classmethoddef category(cls) -> str:- return "ips"+ return "Ip"@classmethoddef input_schema(cls) -> Dict[str, Any]:diff --git a/flowsint-api/app/scanners/organizations/sirene.py b/flowsint-api/app/scanners/organizations/sirene.pydeleted file mode 100644index ba68a064..00000000--- a/flowsint-api/app/scanners/organizations/sirene.py+++ /dev/null@@ -1,159 +0,0 @@-import requests-from typing import List, Dict, Any, TypeAlias, Union-from pydantic import TypeAdapter-from app.scanners.base import Scanner-from app.types.organization import Organization, Organization, Identifier-from app.utils import resolve_type-from app.core.logger import Logger----InputType: TypeAlias = List[Organization]-OutputType: TypeAlias = List[Identifier]--class SireneScanner(Scanner):- """Enrich Organization with data from SIRENE (France only)."""-- @classmethod- def name(cls) -> str:- return "sirene_scanner"-- @classmethod- def category(cls) -> str:- return "Organization"-- @classmethod- def key(cls) -> str:- return "name"-- @classmethod- def input_schema(cls) -> Dict[str, Any]:- adapter = TypeAdapter(InputType)- schema = adapter.json_schema()- # Find the Organization type in $defs- organization_def = schema["$defs"].get("Organization")- if not organization_def:- raise ValueError("Organization type not found in schema")- return {- "type": "Organization",- "properties": [- {"name": prop, "type": resolve_type(info, schema)}- for prop, info in organization_def["properties"].items()- ]- }-- @classmethod- def output_schema(cls) -> Dict[str, Any]:- adapter = TypeAdapter(OutputType)- schema = adapter.json_schema()- type_name, details = list(schema["$defs"].items())[0]- return {- "type": type_name,- "properties": [- {"name": prop, "type": resolve_type(info, schema)}- for prop, info in details["properties"].items()- ]- }-- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- if isinstance(item, str):- cleaned.append(Organization(name=item))- elif isinstance(item, dict) and "name" in item:- cleaned.append(Organization(**item))- elif isinstance(item, Organization):- cleaned.append(item)- return cleaned-- def scan(self, data: InputType) -> OutputType:- results: OutputType = []- for org in data:- try:- enriched = self.query_sirene(org.name)- if enriched:- results.append(enriched)- except Exception as e:- print(f"Error enriching organization {org.name}: {e}")- return results-- def query_sirene(self, name: str) -> Organization:- try:- params = {"q": name, "per_page": 1}- resp = requests.get("https://recherche-entreprises.api.gouv.fr/search", params=params, timeout=10)- resp.raise_for_status()- data = resp.json()- if not data.get("results"):- raise ValueError(f"No match found for {name}")- company = data["results"][0]-- identifiers = []- if siren := company.get("siren"):- identifiers.append(Identifier(type="SIREN", value=siren, country="FR", issued_by="INSEE"))- if siret := company.get("siege", {}).get("siret"):- identifiers.append(Identifier(type="SIRET", value=siret, country="FR", issued_by="INSEE"))-- return Organization(- name=company.get("nom_entreprise") or name,- founding_date=company.get("date_creation"),- country="FR",- description=company.get("categorie_entreprise"),- identifiers=identifiers or None,- )- except Exception as e:- print(f"SIRENE API query failed for {name}: {e}")- return Organization(name=name)-- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- if not self.neo4j_conn:- return results-- for org in results:- self.neo4j_conn.query("""- MERGE (o:organization {name: $name, country: $country})- SET o.founding_date = $founding_date,- o.description = $description,- o.label = $label,- o.caption = $caption,- o.type = $type,- o.sketch_id = $sketch_id- """, {- "name": org.name,- "country": "FR",- "founding_date": org.founding_date,- "description": org.description,- "label": org.name,- "caption": org.name,- "type": "organization",- "sketch_id": self.sketch_id,- })-- for identifier in org.identifiers or []:- self.neo4j_conn.query("""- MERGE (i:identifier {type: $type, value: $value})- SET i.country = $country,- i.issued_by = $issued_by,- i.sketch_id = $sketch_id,- i.type = $type,- i.label = $label,- i.caption = $label- WITH i- MATCH (o:Organization {name: $org_name, country: $org_country})- MERGE (o)-[:HAS_IDENTIFIER {sketch_id: $sketch_id}]->(i)- """, {- "type": identifier.type.lower(),- "label": f"{identifier.value}",- "caption": f"{identifier.value}",- "value": identifier.value,- "country": identifier.country,- "issued_by": identifier.issued_by,- "sketch_id": self.sketch_id,- "org_name": org.name,- "org_country": org.country,- })- Logger.graph_append(self.sketch_id, {"message": f"{org.name}: {identifier.type} {identifier.value} -> {org.name}"})--- return results--diff --git a/flowsint-api/app/scanners/organizations/to_infos.py b/flowsint-api/app/scanners/organizations/to_infos.pynew file mode 100644index 00000000..b45271eb--- /dev/null+++ b/flowsint-api/app/scanners/organizations/to_infos.py@@ -0,0 +1,502 @@+from typing import List, Dict, Any, TypeAlias, Union+from pydantic import TypeAdapter+from app.scanners.base import Scanner+from app.types.organization import Organization+from app.utils import resolve_type+from app.core.logger import Logger+from app.tools.organizations.sirene import SireneTool+++InputType: TypeAlias = List[Organization]+OutputType: TypeAlias = List[Organization]++class OrgToInfosScanner(Scanner):+ """Enrich Organization with data from SIRENE (France only)."""++ @classmethod+ def name(cls) -> str:+ return "to_infos"++ @classmethod+ def category(cls) -> str:+ return "Organization"++ @classmethod+ def key(cls) -> str:+ return "name"++ @classmethod+ def input_schema(cls) -> Dict[str, Any]:+ adapter = TypeAdapter(InputType)+ schema = adapter.json_schema()+ # Find the Organization type in $defs+ organization_def = schema["$defs"].get("Organization")+ if not organization_def:+ raise ValueError("Organization type not found in schema")+ return {+ "type": "Organization",+ "properties": [+ {"name": "name", "type": "string"}+ ]+ }++ @classmethod+ def output_schema(cls) -> Dict[str, Any]:+ adapter = TypeAdapter(OutputType)+ schema = adapter.json_schema()+ # the items property contains the Organization type reference+ items_schema = schema.get("items", {})+ if "$ref" in items_schema:+ # Extract the type name from the $ref (e.g., "#/$defs/Organization" -> "Organization")+ ref_path = items_schema["$ref"]+ type_name = ref_path.split("/")[-1]+ organization_def = schema["$defs"].get(type_name)+ if not organization_def:+ raise ValueError(f"Type {type_name} not found in schema")+ return {+ "type": type_name,+ "properties": [+ {"name": prop, "type": resolve_type(info, schema)}+ for prop, info in organization_def["properties"].items()+ ]+ }+ else:+ raise ValueError("Expected $ref in items schema for List type")+++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ if not isinstance(data, list):+ raise ValueError(f"Expected list input, got {type(data).__name__}")+ cleaned: InputType = []+ for item in data:+ if isinstance(item, str) and str!="":+ cleaned.append(Organization(name=item))+ elif isinstance(item, dict) and "name" in item and item["name"] != "":+ cleaned.append(Organization(**item))+ elif isinstance(item, Organization):+ cleaned.append(item)+ return cleaned++ def scan(self, data: InputType) -> OutputType:++ results: OutputType = []+ for org in data:+ try:+ sirene = SireneTool()+ raw_orgs = sirene.launch(org.name, limit=25)+ if len(raw_orgs)> 0:+ for org_dict in raw_orgs:+ enriched_org = self.enrich_org(org_dict)+ if enriched_org is not None:+ results.append(enriched_org)+ except Exception as e:+ print(f"Error enriching organization {org.name}: {e}")+ return results++ def enrich_org(self, company: Dict) -> Organization:+ try:+ # Extract siege data+ siege = company.get("siege", {})+ # Create PhysicalAddress for siege_geo_adresse if coordinates exist+ siege_geo_adresse = None+ if siege.get("latitude") and siege.get("longitude"):+ from app.types.address import PhysicalAddress+ siege_geo_adresse = PhysicalAddress(+ address=siege.get("adresse", ""),+ city=siege.get("libelle_commune", ""),+ country="FR", # SIRENE is French registry+ zip=siege.get("code_postal", ""),+ latitude=float(siege.get("latitude", 0)),+ longitude=float(siege.get("longitude", 0))+ )++ # Extract dirigeants and convert to Individual objects+ dirigeants = []+ for dirigeant_data in company.get("dirigeants", []):+ from app.types.individual import Individual+ dirigeant = Individual(+ first_name=dirigeant_data.get("prenoms", ""),+ last_name=dirigeant_data.get("nom", ""),+ full_name=f"{dirigeant_data.get('prenoms', '')} {dirigeant_data.get('nom', '')}".strip(),+ birth_date=dirigeant_data.get("date_de_naissance"),+ gender=None # Not available in SIRENE data+ )+ dirigeants.append(dirigeant)++ name = company.get("nom_raison_sociale") or company.get("nom_complet")++ # Ensure we have a valid name+ if not name:+ Logger.error(self.sketch_id, {"message": f"Organization has no valid name: {company}"})+ return None++ return Organization(+ # Basic information+ name=name,+ siren=company.get("siren"),+ nom_complet=company.get("nom_complet"),+ nom_raison_sociale=company.get("nom_raison_sociale"),+ sigle=company.get("sigle"),++ # Establishment information+ nombre_etablissements=company.get("nombre_etablissements"),+ nombre_etablissements_ouverts=company.get("nombre_etablissements_ouverts"),++ # Activity information+ activite_principale=company.get("activite_principale"),+ section_activite_principale=company.get("section_activite_principale"),++ # Company category+ categorie_entreprise=company.get("categorie_entreprise"),+ annee_categorie_entreprise=company.get("annee_categorie_entreprise"),++ # Employment information+ caractere_employeur=company.get("caractere_employeur"),+ tranche_effectif_salarie=company.get("tranche_effectif_salarie"),+ annee_tranche_effectif_salarie=company.get("annee_tranche_effectif_salarie"),++ # Dates+ date_creation=company.get("date_creation"),+ date_fermeture=company.get("date_fermeture"),+ date_mise_a_jour=company.get("date_mise_a_jour"),+ date_mise_a_jour_insee=company.get("date_mise_a_jour_insee"),+ date_mise_a_jour_rne=company.get("date_mise_a_jour_rne"),++ # Legal information+ nature_juridique=company.get("nature_juridique"),+ etat_administratif=company.get("etat_administratif"),+ statut_diffusion=company.get("statut_diffusion"),++ # Siege (Headquarters) information+ siege_activite_principale=siege.get("activite_principale"),+ siege_activite_principale_registre_metier=siege.get("activite_principale_registre_metier"),+ siege_annee_tranche_effectif_salarie=siege.get("annee_tranche_effectif_salarie"),+ siege_adresse=siege.get("adresse"),+ siege_caractere_employeur=siege.get("caractere_employeur"),+ siege_cedex=siege.get("cedex"),+ siege_code_pays_etranger=siege.get("code_pays_etranger"),+ siege_code_postal=siege.get("code_postal"),+ siege_commune=siege.get("commune"),+ siege_complement_adresse=siege.get("complement_adresse"),+ siege_coordonnees=siege.get("coordonnees"),+ siege_date_creation=siege.get("date_creation"),+ siege_date_debut_activite=siege.get("date_debut_activite"),+ siege_date_fermeture=siege.get("date_fermeture"),+ siege_date_mise_a_jour=siege.get("date_mise_a_jour"),+ siege_date_mise_a_jour_insee=siege.get("date_mise_a_jour_insee"),+ siege_departement=siege.get("departement"),+ siege_distribution_speciale=siege.get("distribution_speciale"),+ siege_epci=siege.get("epci"),+ siege_est_siege=siege.get("est_siege"),+ siege_etat_administratif=siege.get("etat_administratif"),+ siege_geo_adresse=siege_geo_adresse,+ siege_geo_id=siege.get("geo_id"),+ siege_indice_repetition=siege.get("indice_repetition"),+ siege_latitude=siege.get("latitude"),+ siege_libelle_cedex=siege.get("libelle_cedex"),+ siege_libelle_commune=siege.get("libelle_commune"),+ siege_libelle_commune_etranger=siege.get("libelle_commune_etranger"),+ siege_libelle_pays_etranger=siege.get("libelle_pays_etranger"),+ siege_libelle_voie=siege.get("libelle_voie"),+ siege_liste_enseignes=siege.get("liste_enseignes"),+ siege_liste_finess=siege.get("liste_finess"),+ siege_liste_id_bio=siege.get("liste_id_bio"),+ siege_liste_idcc=siege.get("liste_idcc"),+ siege_liste_id_organisme_formation=siege.get("liste_id_organisme_formation"),+ siege_liste_rge=siege.get("liste_rge"),+ siege_liste_uai=siege.get("liste_uai"),+ siege_longitude=siege.get("longitude"),+ siege_nom_commercial=siege.get("nom_commercial"),+ siege_numero_voie=siege.get("numero_voie"),+ siege_region=siege.get("region"),+ siege_siret=siege.get("siret"),+ siege_statut_diffusion_etablissement=siege.get("statut_diffusion_etablissement"),+ siege_tranche_effectif_salarie=siege.get("tranche_effectif_salarie"),+ siege_type_voie=siege.get("type_voie"),++ # Dirigeants (Leaders)+ dirigeants=dirigeants if dirigeants else None,++ # Matching establishments+ matching_etablissements=company.get("matching_etablissements"),++ # Financial information+ finances=company.get("finances"),++ # Complements (Additional information)+ complements_collectivite_territoriale=company.get("complements", {}).get("collectivite_territoriale"),+ complements_convention_collective_renseignee=company.get("complements", {}).get("convention_collective_renseignee"),+ complements_liste_idcc=company.get("complements", {}).get("liste_idcc"),+ complements_egapro_renseignee=company.get("complements", {}).get("egapro_renseignee"),+ complements_est_achats_responsables=company.get("complements", {}).get("est_achats_responsables"),+ complements_est_alim_confiance=company.get("complements", {}).get("est_alim_confiance"),+ complements_est_association=company.get("complements", {}).get("est_association"),+ complements_est_bio=company.get("complements", {}).get("est_bio"),+ complements_est_entrepreneur_individuel=company.get("complements", {}).get("est_entrepreneur_individuel"),+ complements_est_entrepreneur_spectacle=company.get("complements", {}).get("est_entrepreneur_spectacle"),+ complements_est_ess=company.get("complements", {}).get("est_ess"),+ complements_est_finess=company.get("complements", {}).get("est_finess"),+ complements_est_organisme_formation=company.get("complements", {}).get("est_organisme_formation"),+ complements_est_qualiopi=company.get("complements", {}).get("est_qualiopi"),+ complements_liste_id_organisme_formation=company.get("complements", {}).get("liste_id_organisme_formation"),+ complements_est_rge=company.get("complements", {}).get("est_rge"),+ complements_est_service_public=company.get("complements", {}).get("est_service_public"),+ complements_est_l100_3=company.get("complements", {}).get("est_l100_3"),+ complements_est_siae=company.get("complements", {}).get("est_siae"),+ complements_est_societe_mission=company.get("complements", {}).get("est_societe_mission"),+ complements_est_uai=company.get("complements", {}).get("est_uai"),+ complements_est_patrimoine_vivant=company.get("complements", {}).get("est_patrimoine_vivant"),+ complements_bilan_ges_renseigne=company.get("complements", {}).get("bilan_ges_renseigne"),+ complements_identifiant_association=company.get("complements", {}).get("identifiant_association"),+ complements_statut_entrepreneur_spectacle=company.get("complements", {}).get("statut_entrepreneur_spectacle"),+ complements_type_siae=company.get("complements", {}).get("type_siae")+ )+ except Exception as e:+ Logger.error(self.sketch_id, {"message": f"Error enriching organization: {e}"})+ return None++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ if not self.neo4j_conn:+ return results++ for org in results:+ # Create or update the organization node with all SIRENE properties+ self.neo4j_conn.query("""+ MERGE (o:Organization {name: $name, country: $country})+ SET o.siren = $siren,+ o.nom_complet = $nom_complet,+ o.nom_raison_sociale = $nom_raison_sociale,+ o.sigle = $sigle,+ o.nombre_etablissements = $nombre_etablissements,+ o.nombre_etablissements_ouverts = $nombre_etablissements_ouverts,+ o.activite_principale = $activite_principale,+ o.section_activite_principale = $section_activite_principale,+ o.categorie_entreprise = $categorie_entreprise,+ o.annee_categorie_entreprise = $annee_categorie_entreprise,+ o.caractere_employeur = $caractere_employeur,+ o.tranche_effectif_salarie = $tranche_effectif_salarie,+ o.annee_tranche_effectif_salarie = $annee_tranche_effectif_salarie,+ o.date_creation = $date_creation,+ o.date_fermeture = $date_fermeture,+ o.date_mise_a_jour = $date_mise_a_jour,+ o.date_mise_a_jour_insee = $date_mise_a_jour_insee,+ o.date_mise_a_jour_rne = $date_mise_a_jour_rne,+ o.nature_juridique = $nature_juridique,+ o.etat_administratif = $etat_administratif,+ o.statut_diffusion = $statut_diffusion,+ o.sketch_id = $sketch_id,+ o.label = $name,+ o.caption = $name,+ o.type = 'organization'+ """, {+ "name": org.name,+ "country": "FR",+ "siren": org.siren,+ "nom_complet": org.nom_complet,+ "nom_raison_sociale": org.nom_raison_sociale,+ "sigle": org.sigle,+ "nombre_etablissements": org.nombre_etablissements,+ "nombre_etablissements_ouverts": org.nombre_etablissements_ouverts,+ "activite_principale": org.activite_principale,+ "section_activite_principale": org.section_activite_principale,+ "categorie_entreprise": org.categorie_entreprise,+ "annee_categorie_entreprise": org.annee_categorie_entreprise,+ "caractere_employeur": org.caractere_employeur,+ "tranche_effectif_salarie": org.tranche_effectif_salarie,+ "annee_tranche_effectif_salarie": org.annee_tranche_effectif_salarie,+ "date_creation": org.date_creation,+ "date_fermeture": org.date_fermeture,+ "date_mise_a_jour": org.date_mise_a_jour,+ "date_mise_a_jour_insee": org.date_mise_a_jour_insee,+ "date_mise_a_jour_rne": org.date_mise_a_jour_rne,+ "nature_juridique": org.nature_juridique,+ "etat_administratif": org.etat_administratif,+ "statut_diffusion": org.statut_diffusion,+ "sketch_id": self.sketch_id,+ })++ # Add SIREN as identifier if available+ if org.siren:+ self.neo4j_conn.query("""+ MERGE (i:Identifier {type: 'siren', value: $value})+ SET i.country = $country,+ i.issued_by = $issued_by,+ i.sketch_id = $sketch_id,+ i.label = $label,+ i.caption = $label+ WITH i+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_IDENTIFIER {sketch_id: $sketch_id}]->(i)+ """, {+ "type": "siren",+ "label": f"SIREN: {org.siren}",+ "caption": f"SIREN: {org.siren}",+ "value": org.siren,+ "country": "FR",+ "issued_by": "INSEE",+ "sketch_id": self.sketch_id,+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: SIREN {org.siren} -> {org.name}"})++ # Add SIRET as identifier if available+ if org.siege_siret:+ self.neo4j_conn.query("""+ MERGE (i:Identifier {type: 'siret', value: $value})+ SET i.country = $country,+ i.issued_by = $issued_by,+ i.sketch_id = $sketch_id,+ i.label = $label,+ i.caption = $label+ WITH i+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_IDENTIFIER {sketch_id: $sketch_id}]->(i)+ """, {+ "type": "siret",+ "label": f"SIRET: {org.siege_siret}",+ "caption": f"SIRET: {org.siege_siret}",+ "value": org.siege_siret,+ "country": "FR",+ "issued_by": "INSEE",+ "sketch_id": self.sketch_id,+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: SIRET {org.siege_siret} -> {org.name}"})++ # Add dirigeants (leaders) as Individual nodes with relationships+ if org.dirigeants:+ for dirigeant in org.dirigeants:+ self.neo4j_conn.query("""+ MERGE (i:Individual {full_name: $full_name})+ SET i.first_name = $first_name,+ i.last_name = $last_name,+ i.birth_date = $birth_date,+ i.gender = $gender,+ i.sketch_id = $sketch_id,+ i.label = $full_name,+ i.caption = $full_name,+ i.type = 'individual'+ WITH i+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_LEADER {sketch_id: $sketch_id}]->(i)+ """, {+ "full_name": dirigeant.full_name,+ "first_name": dirigeant.first_name,+ "last_name": dirigeant.last_name,+ "birth_date": dirigeant.birth_date,+ "gender": dirigeant.gender,+ "sketch_id": self.sketch_id,+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_LEADER -> {dirigeant.full_name}"})++ # Add siege address as PhysicalAddress node if available+ if org.siege_geo_adresse:+ address = org.siege_geo_adresse+ self.neo4j_conn.query("""+ MERGE (a:PhysicalAddress {address: $address, city: $city, country: $country})+ SET a.zip = $zip,+ a.latitude = $latitude,+ a.longitude = $longitude,+ a.sketch_id = $sketch_id,+ a.label = $label,+ a.caption = $caption,+ a.type = 'location'+ WITH a+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_ADDRESS {sketch_id: $sketch_id}]->(a)+ """, {+ "address": address.address,+ "city": address.city,+ "country": address.country,+ "zip": address.zip,+ "latitude": address.latitude,+ "longitude": address.longitude,+ "sketch_id": self.sketch_id,+ "label": f"{address.address}, {address.city}",+ "caption": f"{address.address}, {address.city}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}"})++ # Add siege location as Location node if coordinates are available but no PhysicalAddress+ elif org.siege_latitude and org.siege_longitude:+ self.neo4j_conn.query("""+ MERGE (l:Location {latitude: $latitude, longitude: $longitude})+ SET l.address = $address,+ l.city = $city,+ l.country = $country,+ l.zip = $zip,+ l.sketch_id = $sketch_id,+ l.label = $label,+ l.caption = $caption,+ l.type = 'location'+ WITH l+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:LOCATED_AT {sketch_id: $sketch_id}]->(l)+ """, {+ "latitude": float(org.siege_latitude),+ "longitude": float(org.siege_longitude),+ "address": org.siege_adresse,+ "city": org.siege_libelle_commune,+ "country": "FR",+ "zip": org.siege_code_postal,+ "sketch_id": self.sketch_id,+ "label": f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",+ "caption": f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}"})++ # Add activity codes as Activity nodes+ if org.activite_principale:+ self.neo4j_conn.query("""+ MERGE (a:Activity {code: $code})+ SET a.section = $section,+ a.sketch_id = $sketch_id,+ a.label = $label,+ a.caption = $caption,+ a.type = 'activity'+ WITH a+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_ACTIVITY {sketch_id: $sketch_id}]->(a)+ """, {+ "code": org.activite_principale,+ "section": org.section_activite_principale,+ "sketch_id": self.sketch_id,+ "label": f"Activity: {org.activite_principale}",+ "caption": f"Activity: {org.activite_principale}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}"})++ # Add legal nature as LegalNature node+ if org.nature_juridique:+ self.neo4j_conn.query("""+ MERGE (l:LegalNature {code: $code})+ SET l.sketch_id = $sketch_id,+ l.label = $label,+ l.caption = $caption,+ l.type = 'legal_nature'+ WITH l+ MATCH (o:Organization {name: $org_name, country: $org_country})+ MERGE (o)-[:HAS_LEGAL_NATURE {sketch_id: $sketch_id}]->(l)+ """, {+ "code": org.nature_juridique,+ "sketch_id": self.sketch_id,+ "label": f"Legal Nature: {org.nature_juridique}",+ "caption": f"Legal Nature: {org.nature_juridique}",+ "org_name": org.name,+ "org_country": "FR",+ })+ Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}"})++ return results++diff --git a/flowsint-api/app/scanners/registry.py b/flowsint-api/app/scanners/registry.pyindex 8fd3f4d5..42c8be67 100644--- a/flowsint-api/app/scanners/registry.py+++ b/flowsint-api/app/scanners/registry.py@@ -1,5 +1,4 @@from typing import Dict, Type-from app.scanners.base import Scannerfrom app.scanners.domains.subdomains import SubdomainScannerfrom app.scanners.domains.whois import WhoisScanner@@ -8,19 +7,20 @@ from app.scanners.ips.reverse_resolve import ReverseResolveScannerfrom app.scanners.ips.geolocation import GeolocationScannerfrom app.scanners.socials.maigret import MaigretScannerfrom app.scanners.emails.holehe import HoleheScanner-from app.scanners.organizations.sirene import SireneScannerfrom app.scanners.ips.ip_to_asn import IpToAsnScannerfrom app.scanners.ips.asn_to_cidrs import AsnToCidrsScannerfrom app.scanners.ips.cidr_to_ips import CidrToIpsScannerfrom app.scanners.organizations.org_to_asn import OrgToAsnScannerfrom app.scanners.domains.domain_to_asn import DomainToAsnScanner-from app.scanners.crypto.wallet_to_transactions import WalletAddressToTransactions+from app.scanners.crypto.wallet_to_transactions import CryptoWalletAddressToTransactionsfrom app.scanners.crypto.wallet_to_nfts import WalletAddressToNFTsfrom app.scanners.domains.to_website import DomainToWebsiteScannerfrom app.scanners.websites.to_crawler import WebsiteToCrawlerfrom app.scanners.websites.to_domain import WebsiteToDomainScannerfrom app.scanners.emails.to_gravatar import EmailToGravatarScannerfrom app.scanners.emails.to_leaks import EmailToBreachesScanner+from app.scanners.individuals.to_org import IndividualToOrgScanner+from app.scanners.organizations.to_infos import OrgToInfosScannerclass ScannerRegistry:@@ -68,6 +68,7 @@ class ScannerRegistry:"name": scanner.name(),"module": scanner.__module__,"doc": scanner.__doc__,+ "category": category,"inputs": scanner.input_schema(),"outputs": scanner.output_schema(),})@@ -80,16 +81,19 @@ ScannerRegistry.register(WhoisScanner)ScannerRegistry.register(GeolocationScanner)ScannerRegistry.register(MaigretScanner)ScannerRegistry.register(HoleheScanner)-ScannerRegistry.register(SireneScanner)ScannerRegistry.register(IpToAsnScanner)ScannerRegistry.register(AsnToCidrsScanner)ScannerRegistry.register(CidrToIpsScanner)ScannerRegistry.register(OrgToAsnScanner)ScannerRegistry.register(DomainToAsnScanner)-ScannerRegistry.register(WalletAddressToTransactions)+ScannerRegistry.register(CryptoWalletAddressToTransactions)ScannerRegistry.register(WalletAddressToNFTs)ScannerRegistry.register(DomainToWebsiteScanner)ScannerRegistry.register(WebsiteToCrawler)ScannerRegistry.register(WebsiteToDomainScanner)ScannerRegistry.register(EmailToGravatarScanner)-ScannerRegistry.register(EmailToBreachesScanner)\ No newline at end of file+ScannerRegistry.register(EmailToBreachesScanner)+ScannerRegistry.register(IndividualToOrgScanner)+ScannerRegistry.register(OrgToInfosScanner)+ScannerRegistry.register(IndividualToOrgScanner)+diff --git a/flowsint-api/app/tools/base.py b/flowsint-api/app/tools/base.pynew file mode 100644index 00000000..a5580a5f--- /dev/null+++ b/flowsint-api/app/tools/base.py@@ -0,0 +1,22 @@+from abc import ABC, abstractmethod+from typing import Any++class Tool(ABC):+ @classmethod+ @abstractmethod+ def name(cls) -> str:+ pass++ @classmethod+ @abstractmethod+ def category(cls) -> str:+ pass++ @classmethod+ @abstractmethod+ def description(cls) -> str:+ pass++ @abstractmethod+ def launch(self, value: str, *args, **kwargs) -> Any:+ passdiff --git a/flowsint-api/app/tools/dockertool.py b/flowsint-api/app/tools/dockertool.pynew file mode 100644index 00000000..572df5fe--- /dev/null+++ b/flowsint-api/app/tools/dockertool.py@@ -0,0 +1,61 @@+from docker import from_env+from docker.errors import ImageNotFound, APIError, DockerException+from app.tools.base import Tool++class DockerTool(Tool):+ def __init__(self, image: str, default_tag: str = "latest"):+ self.image = f"{image}:{default_tag}"+ self.client = from_env()++ @classmethod+ def get_image(cls) -> str:+ return cls.image++ def install(self):+ try:+ print(f"[DockerTool] Pulling image: {self.image}")+ self.client.images.pull(self.image)+ except APIError as e:+ raise RuntimeError(f"Failed to pull image {self.image}: {e.explanation}")++ def version(self) -> str:+ try:+ output = self.client.containers.run(+ self.image,+ command="--version",+ remove=True,+ stdout=True,+ stderr=True,+ detach=False,+ tty=False+ )+ return output.decode().strip()+ except Exception as e:+ return f"unknown (error: {str(e)})"++ def is_installed(self) -> bool:+ try:+ self.client.images.get(self.image)+ return True+ except ImageNotFound:+ return False+++ def launch(self, command: str, volumes: dict = None, timeout: int = 30):+ self.install()+ try:+ return self.client.containers.run(+ self.image,+ command=command,+ remove=True,+ stdout=True,+ stderr=True,+ volumes=volumes or {},+ detach=False,+ tty=False,+ network_mode="bridge",+ ).decode()+ except ImageNotFound:+ raise RuntimeError(f"Image {self.image} not found. Did you run install()?")+ except DockerException as e:+ raise RuntimeError(f"Docker error while running {self.image}: {str(e)}")diff --git a/flowsint-api/app/tools/network/asnmap.py b/flowsint-api/app/tools/network/asnmap.pynew file mode 100644index 00000000..a6b1e749--- /dev/null+++ b/flowsint-api/app/tools/network/asnmap.py@@ -0,0 +1,71 @@+import json+from typing import Any, Literal+from app.tools.dockertool import DockerTool++class AsnmapTool(DockerTool):+ image = "projectdiscovery/asnmap"+ default_tag = "latest"++ def __init__(self):+ super().__init__(self.image, self.default_tag)++ @classmethod+ def name(cls) -> str:+ return "asnmap"++ @classmethod+ def description(cls) -> str:+ return "ASN mapping and network reconnaissance tool."++ @classmethod+ def category(cls) -> str:+ return "ASN discovery"++ def install(self) -> None:+ super().install()++ def version(self) -> str:+ try:+ output = self.client.containers.run(+ image=self.image,+ command="--version",+ remove=True,+ stderr=True,+ stdout=True+ )+ output_str = output.decode()+ import re+ match = re.search(r'(v[\d\.]+)', output_str)+ version = match.group(1) if match else "unknown"+ return version+ except Exception as e:+ return f"unknown (error: {str(e)})"++ def update(self) -> None:+ # Pull the latest image+ self.install()++ def is_installed(self) -> bool:+ return super().is_installed()++ def launch(self, item: str, type: Literal["domain", "org", "ip", "asn"] = "domain") -> Any:+ flags = {+ "domain" : '-d',+ 'org': '-org',+ 'ip' : '-i',+ 'asn': '-a'+ }+ flag = flags[type]+ try:+ # Use the -target argument as asnmap expects+ result = super().launch(f"{flag} {item} -silent -json ")+ if result and result != "":+ return json.loads(result)+ else:+ return {}+ except json.JSONDecodeError as e:+ raise RuntimeError(f"Failed to parse JSON output from asnmap: {str(e)}")+ except Exception as e:+ # Try to get more info from the container logs+ raise RuntimeError(f"Error running asnmap: {str(e)}. Output: {getattr(e, 'output', 'No output')}")+diff --git a/flowsint-api/app/tools/network/subfinder.py b/flowsint-api/app/tools/network/subfinder.pynew file mode 100644index 00000000..fffe0105--- /dev/null+++ b/flowsint-api/app/tools/network/subfinder.py@@ -0,0 +1,62 @@+from typing import Any+from app.tools.dockertool import DockerTool+from app.utils import is_valid_domain++class SubfinderTool(DockerTool):+ image = "projectdiscovery/subfinder"+ default_tag = "latest"++ def __init__(self):+ super().__init__(self.image, self.default_tag)++ @classmethod+ def name(cls) -> str:+ return "subfinder"++ @classmethod+ def description(cls) -> str:+ return "Fast passive subdomain enumeration tool."++ @classmethod+ def category(cls) -> str:+ return "Subdomain enumeration"++ def install(self) -> None:+ super().install()++ def version(self) -> str:+ try:+ # subfinder requires input even when checking version, so we provide a dummy domain+ output = self.client.containers.run(+ image=self.image,+ command="--version",+ remove=True,+ stderr=True,+ stdout=True+ )+ output_str = output.decode()+ import re+ match = re.search(r'(v[\d\.]+)', output_str)+ version = match.group(1) if match else "unknown"+ return version+ except Exception as e:+ return f"unknown (error: {str(e)})"+++ def update(self) -> None:+ # Pull the latest image+ self.install()++ def is_installed(self) -> bool:+ return super().is_installed()++ def launch(self, domain: str) -> Any:+ subdomains: set[str] = set()+ command = f"-d {domain}"+ result = super().launch(command)+ for sub in result.split("\n"):+ print(sub)+ if is_valid_domain(sub) and sub.endswith(domain) and sub != domain and not sub.startswith("."):+ subdomains.add(sub)+ return list(subdomains)+diff --git a/flowsint-api/app/tools/organizations/sirene.py b/flowsint-api/app/tools/organizations/sirene.pynew file mode 100644index 00000000..a0f72f27--- /dev/null+++ b/flowsint-api/app/tools/organizations/sirene.py@@ -0,0 +1,35 @@+import json+from typing import Dict++import requests+from app.tools.base import Tool++class SireneTool(Tool):++ @classmethod+ def name(cls) -> str:+ return "sirene"++ @classmethod+ def description(cls) -> str:+ return "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."++ @classmethod+ def category(cls) -> str:+ return "Business intelligence"++ def launch(self, query: str, limit: int=25) -> list[Dict]:+ # Query can be multiple types of entries: full names ("<first> + <last>"), name, geolocation, etc.+ try:+ query = query.replace(" ", "+")+ params = {"q": query, "per_page": limit }+ resp = requests.get("https://recherche-entreprises.api.gouv.fr/search", params=params, timeout=10)+ resp.raise_for_status()+ data = resp.json()+ if len(data.get("results")) == 0:+ raise ValueError(f"No match found for {query}.")+ return data["results"]+ except Exception as e:+ raise RuntimeError(f"Error querying Sirene API: {str(e)}. Output: {getattr(e, 'output', 'No output')}")++diff --git a/flowsint-api/app/types/domain.py b/flowsint-api/app/types/domain.pyindex df2dd52d..0ff3cee8 100644--- a/flowsint-api/app/types/domain.py+++ b/flowsint-api/app/types/domain.py@@ -1,6 +1,9 @@+from typing import Optionalfrom pydantic import BaseModel, Fieldclass Domain(BaseModel):domain: str = Field(..., description="Domain name")+ root: Optional[bool]= Field(True, description="Is root or not")+Domain.model_rebuild()diff --git a/flowsint-api/app/types/organization.py b/flowsint-api/app/types/organization.pyindex 6d89e979..dab1ae09 100644--- a/flowsint-api/app/types/organization.py+++ b/flowsint-api/app/types/organization.py@@ -1,16 +1,114 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+from .individual import Individual+from .address import PhysicalAddress-class Identifier(BaseModel):- type: str = Field(..., description="Type of identifier (e.g., SIREN, SIRET, DUNS, VAT, LEI)")- value: str = Field(..., description="The identifier value")- country: Optional[str] = Field(None, description="Country where the identifier was issued")- issued_by: Optional[str] = Field(None, description="Authority that issued the identifier")--class Organization(BaseModel):+ # Basic informationname: str = Field(..., description="Organization name")- founding_date: Optional[str] = Field(None, description="Date when the organization was founded")- country: Optional[str] = Field(None, description="Country where the organization is based")- description: Optional[str] = Field(None, description="Description of the organization's activities")- identifiers: Optional[List[Identifier]] = Field(None, description="Identifiers of the organization")\ No newline at end of file+ siren: Optional[str] = Field(None, description="SIREN number")+ nom_complet: Optional[str] = Field(None, description="Complete name")+ nom_raison_sociale: Optional[str] = Field(None, description="Company name")+ sigle: Optional[str] = Field(None, description="Acronym")+ # Establishment information+ nombre_etablissements: Optional[int] = Field(None, description="Number of establishments")+ nombre_etablissements_ouverts: Optional[int] = Field(None, description="Number of open establishments")+ # Activity information+ activite_principale: Optional[str] = Field(None, description="Main activity code")+ section_activite_principale: Optional[str] = Field(None, description="Main activity section")+ # Company category+ categorie_entreprise: Optional[str] = Field(None, description="Company category")+ annee_categorie_entreprise: Optional[str] = Field(None, description="Year of company category")+ # Employment information+ caractere_employeur: Optional[str] = Field(None, description="Employer character")+ tranche_effectif_salarie: Optional[str] = Field(None, description="Employee size range")+ annee_tranche_effectif_salarie: Optional[str] = Field(None, description="Year of employee size range")+ # Dates+ date_creation: Optional[str] = Field(None, description="Creation date")+ date_fermeture: Optional[str] = Field(None, description="Closing date")+ date_mise_a_jour: Optional[str] = Field(None, description="Update date")+ date_mise_a_jour_insee: Optional[str] = Field(None, description="INSEE update date")+ date_mise_a_jour_rne: Optional[str] = Field(None, description="RNE update date")+ # Legal information+ nature_juridique: Optional[str] = Field(None, description="Legal nature")+ etat_administratif: Optional[str] = Field(None, description="Administrative status")+ statut_diffusion: Optional[str] = Field(None, description="Diffusion status")+ # Siege (Headquarters) information+ siege_activite_principale: Optional[str] = Field(None, description="Siege main activity code")+ siege_activite_principale_registre_metier: Optional[str] = Field(None, description="Siege main activity in trade register")+ siege_annee_tranche_effectif_salarie: Optional[str] = Field(None, description="Siege year of employee size range")+ siege_adresse: Optional[str] = Field(None, description="Siege address")+ siege_caractere_employeur: Optional[str] = Field(None, description="Siege employer character")+ siege_cedex: Optional[str] = Field(None, description="Siege CEDEX code")+ siege_code_pays_etranger: Optional[str] = Field(None, description="Siege foreign country code")+ siege_code_postal: Optional[str] = Field(None, description="Siege postal code")+ siege_commune: Optional[str] = Field(None, description="Siege municipality code")+ siege_complement_adresse: Optional[str] = Field(None, description="Siege address complement")+ siege_coordonnees: Optional[str] = Field(None, description="Siege coordinates")+ siege_date_creation: Optional[str] = Field(None, description="Siege creation date")+ siege_date_debut_activite: Optional[str] = Field(None, description="Siege activity start date")+ siege_date_fermeture: Optional[str] = Field(None, description="Siege closing date")+ siege_date_mise_a_jour: Optional[str] = Field(None, description="Siege update date")+ siege_date_mise_a_jour_insee: Optional[str] = Field(None, description="Siege INSEE update date")+ siege_departement: Optional[str] = Field(None, description="Siege department code")+ siege_distribution_speciale: Optional[str] = Field(None, description="Siege special distribution")+ siege_epci: Optional[str] = Field(None, description="Siege EPCI code")+ siege_est_siege: Optional[bool] = Field(None, description="Siege is headquarters")+ siege_etat_administratif: Optional[str] = Field(None, description="Siege administrative status")+ siege_geo_adresse: Optional[PhysicalAddress] = Field(None, description="Siege geocoded address")+ siege_geo_id: Optional[str] = Field(None, description="Siege geographic ID")+ siege_indice_repetition: Optional[str] = Field(None, description="Siege repetition index")+ siege_latitude: Optional[str] = Field(None, description="Siege latitude")+ siege_libelle_cedex: Optional[str] = Field(None, description="Siege CEDEX label")+ siege_libelle_commune: Optional[str] = Field(None, description="Siege municipality label")+ siege_libelle_commune_etranger: Optional[str] = Field(None, description="Siege foreign municipality label")+ siege_libelle_pays_etranger: Optional[str] = Field(None, description="Siege foreign country label")+ siege_libelle_voie: Optional[str] = Field(None, description="Siege street label")+ siege_liste_enseignes: Optional[List[str]] = Field(None, description="Siege list of brands")+ siege_liste_finess: Optional[List[str]] = Field(None, description="Siege list of FINESS")+ siege_liste_id_bio: Optional[List[str]] = Field(None, description="Siege list of bio IDs")+ siege_liste_idcc: Optional[List[str]] = Field(None, description="Siege list of IDCC")+ siege_liste_id_organisme_formation: Optional[List[str]] = Field(None, description="Siege list of training organization IDs")+ siege_liste_rge: Optional[List[str]] = Field(None, description="Siege list of RGE")+ siege_liste_uai: Optional[List[str]] = Field(None, description="Siege list of UAI")+ siege_longitude: Optional[str] = Field(None, description="Siege longitude")+ siege_nom_commercial: Optional[str] = Field(None, description="Siege commercial name")+ siege_numero_voie: Optional[str] = Field(None, description="Siege street number")+ siege_region: Optional[str] = Field(None, description="Siege region code")+ siege_siret: Optional[str] = Field(None, description="Siege SIRET number")+ siege_statut_diffusion_etablissement: Optional[str] = Field(None, description="Siege establishment diffusion status")+ siege_tranche_effectif_salarie: Optional[str] = Field(None, description="Siege employee size range")+ siege_type_voie: Optional[str] = Field(None, description="Siege street type")+ # Dirigeants (Leaders) - using Individual type+ dirigeants: Optional[List[Individual]] = Field(None, description="List of leaders")+ # Matching establishments+ matching_etablissements: Optional[List[dict]] = Field(None, description="Matching establishments")+ # Financial information+ finances: Optional[dict] = Field(None, description="Financial information")+ # Complements (Additional information)+ complements_collectivite_territoriale: Optional[str] = Field(None, description="Complements territorial community")+ complements_convention_collective_renseignee: Optional[bool] = Field(None, description="Complements collective agreement informed")+ complements_liste_idcc: Optional[List[str]] = Field(None, description="Complements list of IDCC")+ complements_egapro_renseignee: Optional[bool] = Field(None, description="Complements Egapro informed")+ complements_est_achats_responsables: Optional[bool] = Field(None, description="Complements is responsible purchases")+ complements_est_alim_confiance: Optional[bool] = Field(None, description="Complements is food trust")+ complements_est_association: Optional[bool] = Field(None, description="Complements is association")+ complements_est_bio: Optional[bool] = Field(None, description="Complements is bio")+ complements_est_entrepreneur_individuel: Optional[bool] = Field(None, description="Complements is individual entrepreneur")+ complements_est_entrepreneur_spectacle: Optional[bool] = Field(None, description="Complements is show entrepreneur")+ complements_est_ess: Optional[bool] = Field(None, description="Complements is ESS")+ complements_est_finess: Optional[bool] = Field(None, description="Complements is FINESS")+ complements_est_organisme_formation: Optional[bool] = Field(None, description="Complements is training organization")+ complements_est_qualiopi: Optional[bool] = Field(None, description="Complements is Qualiopi")+ complements_liste_id_organisme_formation: Optional[List[str]] = Field(None, description="Complements list of training organization IDs")+ complements_est_rge: Optional[bool] = Field(None, description="Complements is RGE")+ complements_est_service_public: Optional[bool] = Field(None, description="Complements is public service")+ complements_est_l100_3: Optional[bool] = Field(None, description="Complements is L100-3")+ complements_est_siae: Optional[bool] = Field(None, description="Complements is SIAE")+ complements_est_societe_mission: Optional[bool] = Field(None, description="Complements is mission company")+ complements_est_uai: Optional[bool] = Field(None, description="Complements is UAI")+ complements_est_patrimoine_vivant: Optional[bool] = Field(None, description="Complements is living heritage")+ complements_bilan_ges_renseigne: Optional[bool] = Field(None, description="Complements GES balance informed")+ complements_identifiant_association: Optional[str] = Field(None, description="Complements association identifier")+ complements_statut_entrepreneur_spectacle: Optional[str] = Field(None, description="Complements show entrepreneur status")+ complements_type_siae: Optional[str] = Field(None, description="Complements SIAE type")\ No newline at end of filediff --git a/flowsint-api/app/types/wallet.py b/flowsint-api/app/types/wallet.pyindex 0bce56e0..a2467589 100644--- a/flowsint-api/app/types/wallet.py+++ b/flowsint-api/app/types/wallet.py@@ -1,15 +1,15 @@-from typing import List, Optional+from typing import Optionalfrom pydantic import BaseModel, Field, HttpUrl-class Wallet(BaseModel):+class CryptoWallet(BaseModel):"""Represents a cryptocurrency wallet."""address: str = Field(..., description="Wallet address")node_id: Optional[str] = Field(None, description="Wallet Explorer node ID")-class WalletTransaction(BaseModel):+class CryptoWalletTransaction(BaseModel):"""Represents a cryptocurrency transaction."""- source: Wallet = Field(..., description="Source wallet")- target: Optional[Wallet] = Field(None, description="Target wallet")+ source: CryptoWallet = Field(..., description="Source wallet")+ target: Optional[CryptoWallet] = Field(None, description="Target wallet")hash: Optional[str] = Field(None, description="Transaction hash")value: Optional[float] = Field(None, description="Transaction value in cryptocurrency")amount: Optional[float] = Field(None, description="Transaction amount in cryptocurrency")@@ -35,9 +35,9 @@ class WalletTransaction(BaseModel):error_message: Optional[str] = Field(None, description="Error message if transaction failed")-class NFT(BaseModel):+class CryptoNFT(BaseModel):"""Represents a Non-Fungible Token (NFT) held or minted by a wallet."""- wallet: Wallet = Field(..., description="Source wallet")+ wallet: CryptoWallet = Field(..., description="Source wallet")contract_address: str = Field(..., description="Address of the NFT smart contract (ERC-721/1155)")token_id: str = Field(..., description="Unique token ID of the NFT within the contract")collection_name: Optional[str] = Field(None, description="Name of the NFT collection")@@ -56,6 +56,6 @@ class NFT(BaseModel):# Update forward references-Wallet.model_rebuild()-WalletTransaction.model_rebuild()-NFT.model_rebuild()\ No newline at end of file+CryptoWallet.model_rebuild()+CryptoWalletTransaction.model_rebuild()+CryptoNFT.model_rebuild()\ No newline at end of filediff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.pyindex fc3eac05..f8073c02 100644--- a/flowsint-api/app/utils.py+++ b/flowsint-api/app/utils.py@@ -30,19 +30,57 @@ def is_valid_email(email: str) -> bool:return True-def is_valid_domain(url_or_domain: str) -> str:- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)- hostname = parsed.hostname or url_or_domain+ try:+ parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)+ hostname = parsed.hostname or url_or_domain- if not hostname or "." not in hostname:- return False+ if not hostname or "." not in hostname:+ return False++ if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):+ return False- if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):+ return True+ except Exception as e:return False++def is_root_domain(domain: str) -> bool:+ """+ Determine if a domain is a root domain or subdomain.- return hostname+ Args:+ domain: The domain string to check++ Returns:+ True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)+ """+ try:+ # Remove protocol if present+ if "://" in domain:+ parsed = urlparse(domain)+ domain = parsed.hostname or domain++ # Split by dots+ parts = domain.split('.')++ # Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)+ common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',+ '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']++ # Check if the domain ends with a common country code TLD+ for cc_tld in common_cc_tlds:+ if domain.endswith(cc_tld):+ # For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)+ return len(parts) == 3++ # For regular TLDs, a root domain has 2 parts (e.g., example.com)+ # A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)+ return len(parts) == 2+ except Exception:+ # If we can't parse it, assume it's not a root domain+ return Falsedef is_valid_number(phone: str, region: str = "FR") -> None:"""@@ -125,7 +163,9 @@ def extract_input_schema(name: str, model: Type[BaseModel]) -> Dict[str, Any]:"type": "","properties": []},- "type": "type"+ "type": "type",+ "category": name,+}diff --git a/flowsint-api/out.json b/flowsint-api/out.jsonnew file mode 100644index 00000000..7654a606--- /dev/null+++ b/flowsint-api/out.json@@ -0,0 +1,151 @@+{+ "results": [+ {+ "siren": "821231867",+ "nom_complet": "KARIM TERRACHE",+ "nom_raison_sociale": null,+ "sigle": null,+ "nombre_etablissements": 1,+ "nombre_etablissements_ouverts": 1,+ "siege": {+ "activite_principale": "53.20Z",+ "activite_principale_registre_metier": null,+ "annee_tranche_effectif_salarie": null,+ "adresse": "2 RUE DE LA DISTILLERIE 93500 PANTIN",+ "caractere_employeur": "N",+ "cedex": null,+ "code_pays_etranger": null,+ "code_postal": "93500",+ "commune": "93055",+ "complement_adresse": null,+ "coordonnees": "48.894315,2.406248",+ "date_creation": "2016-07-01",+ "date_debut_activite": "2016-07-01",+ "date_fermeture": null,+ "date_mise_a_jour": null,+ "date_mise_a_jour_insee": "2025-02-09T02:14:38",+ "departement": "93",+ "distribution_speciale": null,+ "epci": "200054781",+ "est_siege": true,+ "etat_administratif": "A",+ "geo_adresse": "2 Rue de la Distillerie 93500 Pantin",+ "geo_id": "93055_2560_00002",+ "indice_repetition": null,+ "latitude": "48.894315",+ "libelle_cedex": null,+ "libelle_commune": "PANTIN",+ "libelle_commune_etranger": null,+ "libelle_pays_etranger": null,+ "libelle_voie": "DE LA DISTILLERIE",+ "liste_enseignes": null,+ "liste_finess": null,+ "liste_id_bio": null,+ "liste_idcc": null,+ "liste_id_organisme_formation": null,+ "liste_rge": null,+ "liste_uai": null,+ "longitude": "2.406248",+ "nom_commercial": null,+ "numero_voie": "2",+ "region": "11",+ "siret": "82123186700010",+ "statut_diffusion_etablissement": "O",+ "tranche_effectif_salarie": null,+ "type_voie": "RUE"+ },+ "activite_principale": "53.20Z",+ "categorie_entreprise": "PME",+ "caractere_employeur": null,+ "annee_categorie_entreprise": "2022",+ "date_creation": "2016-07-01",+ "date_fermeture": null,+ "date_mise_a_jour": "2025-07-01T06:50:35",+ "date_mise_a_jour_insee": "2024-03-22T14:26:06",+ "date_mise_a_jour_rne": "2024-05-19T16:49:33",+ "dirigeants": [+ {+ "nom": "TERRACHE",+ "prenoms": "KARIM",+ "annee_de_naissance": null,+ "date_de_naissance": null,+ "qualite": null,+ "nationalite": null,+ "type_dirigeant": "personne physique"+ }+ ],+ "etat_administratif": "A",+ "nature_juridique": "1000",+ "section_activite_principale": "H",+ "tranche_effectif_salarie": null,+ "annee_tranche_effectif_salarie": null,+ "statut_diffusion": "O",+ "matching_etablissements": [+ {+ "activite_principale": "53.20Z",+ "ancien_siege": false,+ "annee_tranche_effectif_salarie": null,+ "adresse": "2 RUE DE LA DISTILLERIE 93500 PANTIN",+ "caractere_employeur": "N",+ "code_postal": "93500",+ "commune": "93055",+ "date_creation": "2016-07-01",+ "date_debut_activite": "2016-07-01",+ "date_fermeture": null,+ "epci": "200054781",+ "est_siege": true,+ "etat_administratif": "A",+ "geo_id": "93055_2560_00002",+ "latitude": "48.894315",+ "libelle_commune": "PANTIN",+ "liste_enseignes": null,+ "liste_finess": null,+ "liste_id_bio": null,+ "liste_idcc": null,+ "liste_id_organisme_formation": null,+ "liste_rge": null,+ "liste_uai": null,+ "longitude": "2.406248",+ "nom_commercial": null,+ "region": "11",+ "siret": "82123186700010",+ "statut_diffusion_etablissement": "O",+ "tranche_effectif_salarie": null+ }+ ],+ "finances": null,+ "complements": {+ "collectivite_territoriale": null,+ "convention_collective_renseignee": false,+ "liste_idcc": null,+ "egapro_renseignee": false,+ "est_achats_responsables": false,+ "est_alim_confiance": false,+ "est_association": false,+ "est_bio": false,+ "est_entrepreneur_individuel": true,+ "est_entrepreneur_spectacle": false,+ "est_ess": false,+ "est_finess": false,+ "est_organisme_formation": false,+ "est_qualiopi": false,+ "liste_id_organisme_formation": null,+ "est_rge": false,+ "est_service_public": false,+ "est_l100_3": false,+ "est_siae": false,+ "est_societe_mission": false,+ "est_uai": false,+ "est_patrimoine_vivant": false,+ "bilan_ges_renseigne": false,+ "identifiant_association": null,+ "statut_entrepreneur_spectacle": null,+ "type_siae": null+ }+ }+ ],+ "total_results": 1,+ "page": 1,+ "per_page": 1,+ "total_pages": 1+}\ No newline at end of filediff --git a/flowsint-api/requirements.txt b/flowsint-api/requirements.txtindex 3f310bfd..df627d31 100644--- a/flowsint-api/requirements.txt+++ b/flowsint-api/requirements.txt@@ -31,4 +31,5 @@ mistralaibeautifulsoup4lxmlrequests-random-user-agent-spacy\ No newline at end of file+spacy+docker\ No newline at end of filediff --git a/flowsint-api/tests/scanners/domains/subdomains.py b/flowsint-api/tests/scanners/domains/subdomains.pyindex 6ddb6429..15606de4 100644--- a/flowsint-api/tests/scanners/domains/subdomains.py+++ b/flowsint-api/tests/scanners/domains/subdomains.py@@ -1,10 +1,8 @@-import pytestfrom app.scanners.domains.subdomains import SubdomainScannerfrom app.types.domain import Domain, Domainscanner = SubdomainScanner("sketch_123", "scan_123")-def test_preprocess_valid_domains():domains = [Domain(domain="example.com"),@@ -95,11 +93,3 @@ def test_scan_extracts_subdomains(monkeypatch):])print(domains)# assert domains[0].subdomains == expected---def test_schemas():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()- assert input_schema == {'type': 'Domain', 'properties': [{'name': 'domain', 'type': 'string'}, {'name': 'subdomains', 'type': 'array | null'}, {'name': 'ips', 'type': 'array | null'}, {'name': 'whois', 'type': 'Whois | null'}]}- assert output_schema == {'type': 'Domain', 'properties': [{'name': 'domain', 'type': 'string'}, {'name': 'subdomains', 'type': 'array | null'}, {'name': 'ips', 'type': 'array | null'}, {'name': 'whois', 'type': 'Whois | null'}]}-diff --git a/flowsint-api/tests/scanners/organizations/sirene.py b/flowsint-api/tests/scanners/organizations/sirene.pydeleted file mode 100644index f41ee868..00000000--- a/flowsint-api/tests/scanners/organizations/sirene.py+++ /dev/null@@ -1,80 +0,0 @@-import pytest-from app.scanners.organizations.sirene import SireneScanner-from app.types.organization import Organization, Organization--scanner = SireneScanner("sketch_123", "scan_123")--def test_preprocess_valid_names():- data = [- Organization(name="OpenAI"),- {"name": "Inria"},- "OVH"- ]- result = scanner.preprocess(data)- result_names = [org.name for org in result]-- assert result_names == ["OpenAI", "Inria", "OVH"]--def test_preprocess_invalid_entries():- data = [- {"wrong_key": "value"},- 123,- None,- "",- {"name": ""},- ]- result = scanner.preprocess(data)- assert result == []--def test_scan_enriches_organization(monkeypatch):- mock_api_response = {- "etablissement": {- "siren": "123456789",- "denomination": "OpenAI",- "dateCreationEtablissement": "2020-01-01",- "activitePrincipale": "Recherche IA",- "adresseEtablissement": {- "libelleCommuneEtablissement": "Paris"- }- }- }-- class MockResponse:- def __init__(self, json_data):- self._json_data = json_data- self.status_code = 200-- def json(self):- return self._json_data-- def raise_for_status(self):- pass-- def mock_get(url, headers, timeout):- assert "OpenAI" in url- return MockResponse(mock_api_response)-- monkeypatch.setattr("requests.get", mock_get)-- input_data = [Organization(name="OpenAI")]- result = scanner.scan(input_data)-- assert isinstance(result, list)- assert len(result) == 1- org = result[0]- assert isinstance(org, Organization)- assert org.name == "OpenAI"- assert org.founding_date == "2020-01-01"- assert org.country == "France"- assert isinstance(org.identifiers, list)- assert org.identifiers[0].type == "SIREN"- assert org.identifiers[0].value == "123456789"--def test_schema():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()-- assert {'name': 'name', 'type': 'string'} in input_schema- output_keys = [entry['name'] for entry in output_schema]- for field in ['name', 'founding_date', 'country', 'description', 'identifiers']:- assert field in output_keysdiff --git a/flowsint-api/tests/scanners/organizations/to_infos.py b/flowsint-api/tests/scanners/organizations/to_infos.pynew file mode 100644index 00000000..0024e455--- /dev/null+++ b/flowsint-api/tests/scanners/organizations/to_infos.py@@ -0,0 +1,30 @@+from app.scanners.organizations.to_infos import OrgToInfosScanner+from app.types.organization import Organization++scanner = OrgToInfosScanner("sketch_123", "scan_123")++def test_preprocess_valid_names():+ data = [+ Organization(name="OpenAI"),+ {"name": "Inria"},+ "OVH"+ ]+ result = scanner.preprocess(data)+ result_names = [org.name for org in result]++ assert result_names == ["OpenAI", "Inria", "OVH"]++# def test_preprocess_invalid_entries():+# data = [+# {"wrong_key": "value"},+# 123,+# None,+# "",+# {"name": ""},+# ]+# result = scanner.preprocess(data)+# assert result == []++def test_execute():+ scanner.execute(["Karim Terrache"])+ assert True\ No newline at end of filediff --git a/flowsint-api/tests/tools/__init__.py b/flowsint-api/tests/tools/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/flowsint-api/tests/tools/network/__init__.py b/flowsint-api/tests/tools/network/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/flowsint-api/tests/tools/network/asnmap.py b/flowsint-api/tests/tools/network/asnmap.pynew file mode 100644index 00000000..cee24d9a--- /dev/null+++ b/flowsint-api/tests/tools/network/asnmap.py@@ -0,0 +1,38 @@+import re+from typing import Dict+from app.tools.network.asnmap import AsnmapTool++tool = AsnmapTool()++def test_name():+ assert tool.name() == "asnmap"++def test_description():+ assert tool.description() == "ASN mapping and network reconnaissance tool."++def test_category():+ assert tool.category() == "ASN discovery"++def test_image():+ assert tool.get_image() == "projectdiscovery/asnmap"++def test_install():+ tool.install()+ assert tool.is_installed() == True++def test_version():+ tool.install()+ version = tool.version()+ # Check that version follows the expected format: v followed by digits and dots+ assert re.match(r'^v[\d\.]+$', version)++def test_launch_wrong_type():+ import pytest+ with pytest.raises(KeyError, match="'domains'"):+ tool.launch("alliage.io", 'domains')++def test_launch():+ assert True+ results = tool.launch("alliage.io", 'domain')+ assert isinstance(results, Dict)+\ No newline at end of filediff --git a/flowsint-api/tests/tools/network/subfinder.py b/flowsint-api/tests/tools/network/subfinder.pynew file mode 100644index 00000000..444c3444--- /dev/null+++ b/flowsint-api/tests/tools/network/subfinder.py@@ -0,0 +1,32 @@+import re+from app.tools.network.subfinder import SubfinderTool++tool = SubfinderTool()++def test_name():+ assert tool.name() == "subfinder"++def test_description():+ assert tool.description() == "Fast passive subdomain enumeration tool."++def test_category():+ assert tool.category() == "Subdomain enumeration"++def test_image():+ assert tool.get_image() == "projectdiscovery/subfinder"++def test_install():+ tool.install()+ assert tool.is_installed() == True++def test_version():+ tool.install()+ version = tool.version()+ # Check that version follows the expected format: v followed by digits and dots+ assert re.match(r'^v[\d\.]+$', version)++def test_launch():+ results = tool.launch("alliage.io")+ assert isinstance(results, list)+ assert all(isinstance(item, str) for item in results)+\ No newline at end of filediff --git a/flowsint-api/tests/tools/organizations/__init__.py b/flowsint-api/tests/tools/organizations/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/flowsint-api/tests/tools/organizations/sirene.py b/flowsint-api/tests/tools/organizations/sirene.pynew file mode 100644index 00000000..eb3f79be--- /dev/null+++ b/flowsint-api/tests/tools/organizations/sirene.py@@ -0,0 +1,29 @@+import re+from typing import Dict+from app.tools.organizations.sirene import SireneTool++tool = SireneTool()++def test_name():+ assert tool.name() == "sirene"++def test_description():+ assert tool.description() == "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."++def test_category():+ assert tool.category() == "Business intelligence"++def test_launch_org():+ results = tool.launch("blablacar", 1)+ assert isinstance(results, list)+ assert all(isinstance(item, Dict) for item in results)++def test_launch_person():+ results = tool.launch("Karim+Terrache", 1)+ assert isinstance(results, list)+ assert all(isinstance(item, Dict) for item in results)++def test_launch_person_space_format():+ results = tool.launch("Karim Terrache", 1)+ assert isinstance(results, list)+ assert all(isinstance(item, Dict) for item in results)diff --git a/flowsint-app/.gitignore b/flowsint-app/.gitignoreindex 42bd71b3..1170717c 100644--- a/flowsint-app/.gitignore+++ b/flowsint-app/.gitignore@@ -1,5 +1,136 @@-node_modules-dist+# Logs+logs+*.log+npm-debug.log*+yarn-debug.log*+yarn-error.log*+lerna-debug.log*+.pnpm-debug.log*++# Diagnostic reports (https://nodejs.org/api/report.html)+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json++# Runtime data+pids+*.pid+*.seed+*.pid.lock++# Directory for instrumented libs generated by jscoverage/JSCover+lib-cov++# Coverage directory used by tools like istanbul+coverage+*.lcov++# nyc test coverage+.nyc_output++# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)+.grunt++# Bower dependency directory (https://bower.io/)+bower_components++# node-waf configuration+.lock-wscript++# Compiled binary addons (https://nodejs.org/api/addons.html)+build/Release++# Dependency directories+node_modules/+jspm_packages/++# Snowpack dependency directory (https://snowpack.dev/)+web_modules/++# TypeScript cache+*.tsbuildinfo++# Optional npm cache directory+.npm++# Optional eslint cache+.eslintcache++# Optional stylelint cache+.stylelintcache++# Microbundle cache+.rpt2_cache/+.rts2_cache_cjs/+.rts2_cache_es/+.rts2_cache_umd/++# Optional REPL history+.node_repl_history++# Output of 'npm pack'+*.tgz++# Yarn Integrity file+.yarn-integrity++# dotenv environment variable files+.env+.env.development.local+.env.test.local+.env.production.local+.env.local++# parcel-bundler cache (https://parceljs.org/)+.cache+.parcel-cache++# Next.js build output+.nextout-.DS_Store-*.log*++# Nuxt.js build / generate output+.nuxt+dist++# Gatsby files+.cache/+# Comment in the public line in if your project uses Gatsby and not Next.js+# https://nextjs.org/blog/next-9-1#public-directory-support+# public++# vuepress build output+.vuepress/dist++# vuepress v2.x temp and cache directory+.temp+.cache++# vitepress build output+**/.vitepress/dist++# vitepress cache directory+**/.vitepress/cache++# Docusaurus cache and generated files+.docusaurus++# Serverless directories+.serverless/++# FuseBox cache+.fusebox/++# DynamoDB Local files+.dynamodb/++# TernJS port file+.tern-port++# Stores VSCode versions used for testing VSCode extensions+.vscode-test++# yarn v2+.yarn/cache+.yarn/unplugged+.yarn/build-state.yml+.yarn/install-state.gz+.pnp.*diff --git a/flowsint-app/src/renderer/index.html b/flowsint-app/src/renderer/index.htmlindex 153b424e..c41e6577 100644--- a/flowsint-app/src/renderer/index.html+++ b/flowsint-app/src/renderer/index.html@@ -1,11 +1,11 @@<!DOCTYPE html><html lang="en" theme="dark"><head>- <script+ <!-- <scriptcrossOrigin="anonymous"src="//unpkg.com/react-scan/dist/auto.global.js">- </script>+ </script> --><meta charset="UTF-8" /><meta name="viewport" content="width=device-width, initial-scale=1.0" />diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsxindex f9cfb439..e83e7116 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx@@ -128,7 +128,7 @@ const GraphPanel = ({ graphData, isLoading, isRefetching }: GraphPanelProps) =><WallEditor isRefetching={isRefetching} isLoading={loading} />)} */}{/* <Graph /> */}- {nodes?.length > 500 ? (+ {nodes?.length > 5000 ? (<Graph />) : (view === "force" ? <GraphReactForce /> :diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsxindex 05cef933..5a8ee02f 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsx@@ -12,7 +12,7 @@ interface GraphReactForceProps {style?: React.CSSProperties;}-const NODE_COUNT_THRESHOLD = 1500;+const NODE_COUNT_THRESHOLD = 500;const GraphReactForce: React.FC<GraphReactForceProps> = () => {const nodes = useGraphStore(s => s.nodes) as GraphNode[];@@ -25,7 +25,11 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {// const currentNode = useGraphStore(s => s.currentNode);const setActions = useGraphControls(s => s.setActions);const [menu, setMenu] = useState<any>(null);- const shouldUseSimpleRendering = useMemo(() => nodes.length > NODE_COUNT_THRESHOLD, [nodes.length]);+ const [currentZoom, setCurrentZoom] = useState(1);+ const shouldUseSimpleRendering = useMemo(() => {+ // Use simple rendering for large node counts or when zoomed out+ return nodes.length > NODE_COUNT_THRESHOLD || currentZoom < 2.5;+ }, [nodes.length, currentZoom]);// Transform data for Force Graphconst graphData = useMemo(() => {@@ -106,10 +110,11 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {const size = node.nodeSize;const type = node.nodeType as ItemType;+ // Use simple rendering for large node countsif (shouldUseSimpleRendering) {// Simple circle rendering for large node countsctx.beginPath();- ctx.arc(node.x, node.y, size / 2, 0, 2 * Math.PI);+ ctx.arc(node.x, node.y, size * .65, 0, 2 * Math.PI);ctx.fillStyle = node.nodeColor;ctx.fill();} else {@@ -126,7 +131,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {if (globalScale > 3) {const label = node.nodeLabel || node.label || node.id;if (label) {- const fontSize = Math.max(2, size * 0.2);+ const fontSize = Math.max(2, size * 0.4);ctx.font = `${fontSize}px -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif`;const textWidth = ctx.measureText(label).width;const padding = 2;@@ -140,7 +145,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {const bgY = node.y + size / 2 + 1;ctx.beginPath();- ctx.roundRect(bgX, bgY, bgWidth, bgHeight, 2);+ ctx.roundRect(bgX, bgY, bgWidth, bgHeight, .75);ctx.fill();ctx.stroke();@@ -239,7 +244,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {return (<div className="relative h-full grow w-full bg-background">- <div ref={containerRef} style={{ width: '100%', height: '100%', minHeight: 300 }}>+ <div ref={containerRef} style={{ width: '100%', height: '100%', minHeight: 300, minWidth: 300 }}><ForceGraph2Dref={graphRef}width={dimensions.width}@@ -288,6 +293,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {}}}backgroundColor="transparent"+ onZoom={(zoom) => setCurrentZoom(zoom.k)}/>{menu && <ContextMenuonClick={handleBackgroundClick}diff --git a/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx b/flowsint-app/src/renderer/src/components/graphs/new-actions.tsxindex b7ec2310..c9ee3a8e 100644--- a/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx@@ -45,8 +45,14 @@ export default function ActionDialog({ children, setCurrentNode }: ActionDialogPreturn}const label_key = currentNodeType.fields.find((f: FormField) => f.name === currentNodeType.label_key)?.name || currentNodeType.fields[0].name- const label = data[label_key as keyof typeof data]const type = currentNodeType.type+ let label = data[label_key as keyof typeof data]+ if (type === "Individual") {+ data.full_name = `${data.first_name} ${data.last_name}`+ label = data.full_name+ }++ // Quick fix in the case where it's an individual, we want to add the fullnameconst newNode = {type: "custom",label: label,diff --git a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsxindex 095d929f..82612f4d 100644--- a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx@@ -6,7 +6,6 @@ import { Button } from "@/components/ui/button"import { Badge } from "@/components/ui/badge"import { TypeBadge } from "@/components/type-badge"import { Search, FilterIcon, XIcon } from "lucide-react"-import { usePlatformIcons } from "@/hooks/use-platform-icons"import { Input } from "@/components/ui/input"import { actionItems } from "@/lib/action-items"import { cn } from "@/lib/utils"@@ -24,61 +23,22 @@ import { Checkbox } from "@/components/ui/checkbox"// Hauteur fixe pour chaque item (en pixels)const ITEM_HEIGHT = 40-// Mémoiser le composant SocialNode-const SocialNode = memo(- ({- node,- setCurrentNode,- onCheckboxChange,- isNodeChecked,- }: {- node: any- setCurrentNode: (node: GraphNode) => void- onCheckboxChange: (node: GraphNode, checked: boolean) => void- isNodeChecked: (nodeId: string) => boolean- }) => {- const platformsIcons = usePlatformIcons()- const platformIcon = useMemo(() => {- // @ts-ignore- return platformsIcons?.[node?.data?.platform]?.icon- }, [platformsIcons, node?.data?.platform])-- const handleClick = useCallback(() => setCurrentNode(node), [node, setCurrentNode])-- const handleCheckboxChange = useCallback(- (checked: boolean) => {- onCheckboxChange(node, checked)- },- [node, onCheckboxChange],- )-- return (- <div className="flex items-center border-b h-full">- <div className="pl-2">- <Checkbox checked={isNodeChecked(node.id)} onCheckedChange={handleCheckboxChange} className="mr-1" />- </div>- <Button- variant={"ghost"}- className={cn(- "flex-1 flex items-center justify-start p-4 !py-5 rounded-none text-left border-l-2 border-l-transparent h-full gap-2"- )}- onClick={handleClick}- >- <Badge variant="secondary" className="h-7 w-7 p-0 rounded-full flex-shrink-0">- {platformIcon}- </Badge>- <div className="min-w-0 grow truncate text-ellipsis">- {node?.data?.username || <span className="italic opacity-60">Unknown</span>}- </div>- <div className="flex-shrink-0">- <TypeBadge type={node?.data?.type as string} />- </div>- </Button>- </div>- )- },-)-+// Function to extract all leaf-level node types from actionItems+const getAllNodeTypes = () => {+ const types: string[] = []+ actionItems.forEach(item => {+ if (item.children) {+ item.children.forEach(child => {+ if (child.type && !types.includes(child.type)) {+ types.push(child.type)+ }+ })+ } else if (item.type && !types.includes(item.type)) {+ types.push(item.type)+ }+ })+ return types.sort()+}// Mémoiser le composant NodeRendererconst NodeRenderer = memo(({@@ -100,17 +60,6 @@ const NodeRenderer = memo([node, onCheckboxChange],)- if (node.data?.type === "social") {- return (- <SocialNode- setCurrentNode={setCurrentNode}- node={node}- onCheckboxChange={onCheckboxChange}- isNodeChecked={isNodeChecked}- />- )- }-return (<div className="flex items-center hover:bg-muted border-b h-full"><div className="pl-2">@@ -171,7 +120,9 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?://@ts-ignorenode?.data?.label?.toLowerCase().includes(searchText) || node?.id?.toLowerCase().includes(searchText)- const matchesFilter = !filters || filters.includes(node?.data?.type as string)+ const matchesFilter = !filters || filters.some(filter =>+ filter.toLowerCase() === (node?.data?.type as string)?.toLowerCase()+ )return matchesSearch && matchesFilter})@@ -212,6 +163,20 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:[selectedNodes],)+ const handleCheckAll = useCallback(+ (checked: boolean) => {+ if (!checked) {+ setSelectedNodes([])+ return+ }+ if (filters?.length || searchQuery) {+ setSelectedNodes(filteredNodes)+ }+ else setSelectedNodes(nodes)+ },+ [nodes, filteredNodes, filters, searchQuery, selectedNodes, setSelectedNodes],+ )+const handleCheckboxChange = useCallback((node: GraphNode, checked: boolean) => {if (checked) {@@ -227,7 +192,11 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:<div className="overflow-hidden bg-card h-full flex flex-col w-full !p-0 !m-0">{/* Header fixe */}<div className="sticky border-b top-0 p-2 bg-card z-10 flex-shrink-0">+<div className="flex items-center gap-2">+ <div>+ <Checkbox onCheckedChange={handleCheckAll} className="mr-1" />+ </div><Badge className="h-7" variant={"outline"}><span className="font-semibold">{(filters?.length || searchQuery) ? `${filteredNodes?.length} / ${nodes?.length || 0}` : nodes?.length || 0}</span>{" "}@@ -275,13 +244,13 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:<DropdownMenuItem className={cn(filters == null && "bg-primary")} onClick={() => toggleFilter(null)}>All</DropdownMenuItem>- {actionItems.map((item) => (+ {getAllNodeTypes().map((type) => (<DropdownMenuItem- className={cn(filters?.includes(item?.type) && "bg-primary/30")}- key={item.id}- onClick={() => toggleFilter(item?.type)}+ className={cn(filters?.includes(type) && "bg-primary/30")}+ key={type}+ onClick={() => toggleFilter(type)}>- {item?.type || "unknown"}+ {type || "unknown"}</DropdownMenuItem>))}</DropdownMenuContent>diff --git a/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx b/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsxindex 6a972862..312e64b6 100644--- a/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx+++ b/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx@@ -41,14 +41,6 @@ const SecondaryNavigation = () => {}- if (investigationId && id && type === "wall") {- return (- <div className="grow">- (WALL) THis will show: entities, items- </div>- )- }-if (pathname.startsWith("/dashboard/transforms")) {return (<div className="grow w-full h-full overflow-x-hidden">diff --git a/flowsint-app/src/renderer/src/components/layout/status-bar.tsx b/flowsint-app/src/renderer/src/components/layout/status-bar.tsxindex e95c3090..3694019b 100644--- a/flowsint-app/src/renderer/src/components/layout/status-bar.tsx+++ b/flowsint-app/src/renderer/src/components/layout/status-bar.tsx@@ -3,25 +3,27 @@ import { Terminal, Trash, Unlock } from "lucide-react"import { ModeToggle } from "../mode-toggle"import { useLayoutStore } from "@/stores/layout-store"import Legend from "../graphs/legend"-import { Link } from "@tanstack/react-router"+import { Link, useParams } from "@tanstack/react-router"import InfoDialog from "./info"export function StatusBar() {+ const { id: sketch_id } = useParams({ strict: false })const isOpenConsole = useLayoutStore(s => s.isOpenConsole)const toggleConsole = useLayoutStore(s => s.toggleConsole)return (<div className="flex items-center bg-card h-8 border-t px-2 text-xs text-muted-foreground"><div className="flex items-center gap-2">- <Button- variant="ghost"- size="sm"- className="h-6 gap-1 text-xs hover:bg-accent"- onClick={toggleConsole}- >- <Terminal strokeWidth={1.4} className="h-3 w-3" />- <span>Console {isOpenConsole ? '(Open)' : ''}</span>- </Button>+ {sketch_id &&+ <Button+ variant="ghost"+ size="sm"+ className="h-6 gap-1 text-xs hover:bg-accent"+ onClick={toggleConsole}+ >+ <Terminal strokeWidth={1.4} className="h-3 w-3" />+ <span>Console {isOpenConsole ? '(Open)' : ''}</span>+ </Button>}</div><div className="flex-1"></div><div className="flex items-center gap-2">@@ -39,6 +41,6 @@ export function StatusBar() {<InfoDialog /><ModeToggle /></div>- </div>+ </div >)}diff --git a/flowsint-app/src/renderer/src/components/transforms/editor.tsx b/flowsint-app/src/renderer/src/components/transforms/editor.tsxindex d6add171..6055adf0 100644--- a/flowsint-app/src/renderer/src/components/transforms/editor.tsx+++ b/flowsint-app/src/renderer/src/components/transforms/editor.tsx@@ -59,6 +59,7 @@ const defaultMarkerEnd: EdgeMarker = {}const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform }: TransformEditorProps) => {+ // #### React Flow and UI State ####const { fitView, zoomIn, zoomOut, setCenter } = useReactFlow()const reactFlowWrapper = useRef<HTMLDivElement>(null)const [reactFlowInstance, setReactFlowInstance] = useState<any>(null)@@ -67,10 +68,14 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transformconst { transformId } = useParams({ strict: false })const [showModal, setShowModal] = useState(false)const hasInitialized = useRef(false)++ // #### Simulation State ####const [isSimulating, setIsSimulating] = useState(false)const [currentStepIndex, setCurrentStepIndex] = useState(0)const [simulationSpeed, setSimulationSpeed] = useState(1000) // ms per stepconst [transformBranches, setTransformsBranches] = useState<any[]>([])++ // #### Transform Store State ####const nodes = useTransformStore(state => state.nodes)const edges = useTransformStore(state => state.edges)const selectedNode = useTransformStore(state => state.selectedNode)@@ -84,6 +89,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transformconst setLoading = useTransformStore(state => state.setLoading)const deleteNode = useTransformStore(state => state.deleteNode)+ // #### Initialization Effects ####// Initialize store with initial datauseEffect(() => {if (!hasInitialized.current && (initialNodes.length > 0 || initialEdges.length > 0)) {@@ -106,6 +112,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform}}, [selectedNode, reactFlowInstance, setCenter])+ // #### Drag and Drop Handlers ####const onDragOver = useCallback((event: React.DragEvent<HTMLDivElement>) => {event.preventDefault()event.dataTransfer.dropEffect = "move"@@ -160,6 +167,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform[reactFlowInstance, nodes, setNodes, setCenter],)+ // #### Node Interaction Handlers ####const onNodeClick: NodeMouseHandler = useCallback((_: React.MouseEvent, node: Node) => {const typedNode = node as TransformNode@@ -178,6 +186,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transformsetSelectedNode(null)}, [setSelectedNode])+ // #### Layout and Node Management ####const onLayout = useCallback(() => {// Wait for nodes to be measured before running layoutsetTimeout(() => {@@ -210,6 +219,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform}}, [selectedNode, deleteNode])+ // #### Transform CRUD Operations ####const saveTransform = useCallback(async (name: string, description: string) => {setLoading(true)@@ -274,6 +284,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform}}, [transformId, confirm, setLoading])+ // #### Flow Computation ####const handleComputeFlow = useCallback(async () => {if (!transformId) {toast.error("Save the transform first to compute the flow.")@@ -297,6 +308,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform}}, [transformId, nodes, edges])+ // #### Simulation State Management ####// Update the updateNodeState function with proper typesconst updateNodeState = useCallback((nds: TransformNode[], nodeId: string, state: 'pending' | 'processing' | 'completed' | 'error') => {return nds.map((node) => {@@ -313,7 +325,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform})}, [])- // In the simulation effect+ // #### Simulation Effect ####useEffect(() => {if (!isSimulating || loading) return@@ -372,6 +384,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transformreturn () => clearTimeout(timer)}, [isSimulating, currentStepIndex, simulationSpeed, loading, transformBranches, updateNodeState])+ // #### Simulation Control Functions ####const startSimulation = () => {// Reset all nodes to pending statesetNodes((nds: TransformNode[]) =>@@ -465,6 +478,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform)}+ // #### Render ####return (<><div ref={reactFlowWrapper} className="w-full h-full bg-background">@@ -541,6 +555,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transformTransformEditor.displayName = "TransformEditor"+// #### Main Component ####function TransformEditor({initialEdges = [],initialNodes = [],@@ -568,11 +583,11 @@ function TransformEditor({)}-// Update the NodePanel type+// #### Node Panel Component ####const NodePanel = memo(({ node, onDelete }: { node: TransformNode; onDelete: () => void }) => (<Panel position="bottom-right" className="bg-card p-3 rounded-md shadow-md border w-72"><div className="flex justify-between items-start mb-2">- <h3 className="font-medium">{node.data.class_name}</h3>+ <h3 className="font-medium">{node.data.name}</h3><Button variant="ghost" size="icon" className="h-6 w-6" onClick={onDelete}><TrashIcon className="h-4 w-4" /></Button>diff --git a/flowsint-app/src/renderer/src/components/transforms/raw-material.tsx b/flowsint-app/src/renderer/src/components/transforms/raw-material.tsxindex fa368d2e..eb8dcd91 100644--- a/flowsint-app/src/renderer/src/components/transforms/raw-material.tsx+++ b/flowsint-app/src/renderer/src/components/transforms/raw-material.tsx@@ -1,6 +1,5 @@import { useMemo, useState } from "react"import { Search, X } from "lucide-react"-import { categoryColors } from "./scanner-data"import ScannerItem, { type Scanner } from "./scanner-item"import { Input } from "../ui/input"import { Button } from "../ui/button"@@ -12,7 +11,7 @@ import { SkeletonList } from "../shared/skeleton-list"export default function RawMaterial() {const { data: materials, isLoading, error } = useQuery({- queryKey: ["investigations"],+ queryKey: ["raw_material"],queryFn: transformService.getRawMaterial,})const [searchTerm, setSearchTerm] = useState<string>("")@@ -76,8 +75,7 @@ export default function RawMaterial() {<ScannerItemkey={scanner.name}scanner={scanner}- category={category}- color={categoryColors[category] || categoryColors[scanner.type] || "#94a3b8"}+ category={scanner.category}/>))}</div>diff --git a/flowsint-app/src/renderer/src/components/transforms/scanner-item.tsx b/flowsint-app/src/renderer/src/components/transforms/scanner-item.tsxindex 4cf34b3a..d5c65fda 100644--- a/flowsint-app/src/renderer/src/components/transforms/scanner-item.tsx+++ b/flowsint-app/src/renderer/src/components/transforms/scanner-item.tsx@@ -6,7 +6,7 @@ import { Button } from "@/components/ui/button"import { Info, GripVertical } from "lucide-react"import { TooltipProvider } from "@/components/ui/tooltip"import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"-import { getScannerColor } from "./scanner-data"+import { useNodesDisplaySettings } from "@/stores/node-display-settings"// Types for the scanner based on the new structureexport interface ScannerProperty {@@ -21,6 +21,7 @@ export interface ScannerIO {export interface Scanner {class_name: string+ category: stringname: stringmodule: stringdoc: string | null@@ -32,7 +33,6 @@ export interface Scanner {interface ScannerItemProps {scanner: Scannercategory: string- color: string}// Custom equality function for ScannerItem@@ -42,14 +42,16 @@ function areEqual(prevProps: ScannerItemProps, nextProps: ScannerItemProps) {prevProps.scanner.name === nextProps.scanner.name &&prevProps.scanner.module === nextProps.scanner.module &&prevProps.scanner.doc === nextProps.scanner.doc &&- prevProps.category === nextProps.category &&- prevProps.color === nextProps.color+ prevProps.category === nextProps.category)}// Memoized scanner item component for the sidebarconst ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {- const borderColor = getScannerColor(scanner.type, category)+ const colors = useNodesDisplaySettings(s => s.colors)+ const borderInputColor = colors[scanner.inputs.type.toLowerCase()]+ const borderOutputColor = colors[scanner.outputs.type.toLowerCase()]+const [isDialogOpen, setIsDialogOpen] = useState(false)// Handler for drag start - using useCallback to prevent recreation on each render@@ -69,22 +71,24 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {draggableonDragStart={onDragStart}className="p-3 rounded-md relative w-full overflow-hidden cursor-grab bg-card border hover:shadow-md transition-all group"- style={{ borderLeftWidth: "4px", borderLeftColor: borderColor, cursor: "grab" }}+ style={{ borderLeftWidth: "8px", borderRightWidth: "8px", borderLeftColor: borderInputColor ?? borderOutputColor, borderRightColor: borderOutputColor, cursor: "grab" }}>- <div className="flex justify-between items-start">- <div className="flex items-start gap-2">- <GripVertical className="h-5 w-5 text-muted-foreground/50 group-hover:text-muted-foreground transition-colors" />- <div className="space-y-1">- <h3 className="text-sm font-medium">{scanner.class_name}</h3>+ <div className="flex justify-between grow items-start">+ <div className="flex items-start gap-2 grow truncate text-ellipsis">+ <div>+ <GripVertical className="h-5 w-5 text-muted-foreground/50 group-hover:text-muted-foreground transition-colors" />+ </div>+ <div className="space-y-1 truncate">+ <h3 className="text-sm font-medium truncate text-ellipsis">{scanner.class_name}</h3>{scanner.type !== "type" &&<div className="mt-2 text-xs"><div className="flex items-center gap-1"><span className="font-medium">Input:</span>- <span className="text-muted-foreground">{scanner.inputs.type}</span>+ <span className="text-muted-foreground truncate text-ellipsis">{scanner.inputs.type}</span></div><div className="flex items-center gap-1"><span className="font-medium">Output:</span>- <span className="text-muted-foreground">{scanner.outputs.type}</span>+ <span className="text-muted-foreground truncate text-ellipsis">{scanner.outputs.type}</span></div></div>}</div>@@ -92,30 +96,30 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {<div className="flex items-center gap-1"><DialogTrigger asChild><Button variant="ghost" size="icon" className="h-6 w-6">- <Info className="h-4 w-4" />+ <Info className="h-4 w-4 opacity-60" strokeWidth={1.5} /></Button></DialogTrigger></div></div></div>- <DialogContent className="sm:max-w-[425px]">+ <DialogContent className="sm:max-w-[725px] max-h-[90vh] overflow-y-auto"><DialogHeader><DialogTitle className="flex items-center gap-2">- <div className="w-1 h-6 rounded-full" style={{ backgroundColor: borderColor }} />+ <div className="w-1 h-6 rounded-full" style={{ backgroundColor: borderInputColor }} />{scanner.class_name}</DialogTitle></DialogHeader><div className="grid gap-4 py-4"><div className="space-y-2">- <h4 className="font-medium text-sm" style={{ color: borderColor }}>Description</h4>+ <h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Description</h4><p className="text-sm text-muted-foreground">{scanner.doc || "No description available"}</p></div><div className="space-y-2">- <h4 className="font-medium text-sm" style={{ color: borderColor }}>Module</h4>+ <h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Module</h4><p className="text-sm text-muted-foreground">{scanner.module}</p></div><div className="space-y-2">- <h4 className="font-medium text-sm" style={{ color: borderColor }}>Input Properties</h4>+ <h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Input Properties</h4><div className="space-y-1">{scanner.inputs.properties.map((prop, index) => (<div key={index} className="text-sm">@@ -126,7 +130,7 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {</div></div><div className="space-y-2">- <h4 className="font-medium text-sm" style={{ color: borderColor }}>Output Properties</h4>+ <h4 className="font-medium text-sm" style={{ color: borderOutputColor }}>Output Properties</h4><div className="space-y-1">{scanner.outputs.properties.map((prop, index) => (<div key={index} className="text-sm">diff --git a/flowsint-app/src/renderer/src/components/transforms/scanner-node.tsx b/flowsint-app/src/renderer/src/components/transforms/scanner-node.tsxindex ecf054c7..76485ab3 100644--- a/flowsint-app/src/renderer/src/components/transforms/scanner-node.tsx+++ b/flowsint-app/src/renderer/src/components/transforms/scanner-node.tsx@@ -3,8 +3,9 @@ import { Position } from "@xyflow/react"import { LabeledHandle } from "../ui/labeled-handle"import { Badge } from "../ui/badge"import { BaseNodeSchema } from "../ui/base-node"-import { getScannerColor } from "./scanner-data"import { NodeStatusIndicator } from "../ui/node-status-indicator"+import { useNodesDisplaySettings } from "@/stores/node-display-settings"+import { useIcon } from "@/hooks/use-icon"// Types for the scanner node based on the new structureexport interface ScannerNodeData extends Record<string, unknown> {@@ -70,8 +71,13 @@ const getStateColor = (state?: string) => {// Memoized scanner node component with custom equality checkconst ScannerNode = memo(({ data, selected, isConnectable }: ScannerNodeProps) => {- const color = getScannerColor(data.type, data.category)+ const colors = useNodesDisplaySettings(s => s.colors)+ const inputColor = colors[data.inputs.type.toLowerCase()]+ const outputColor = colors[data.outputs.type.toLowerCase()]const opacity = data.computationState === "pending" ? 0.5 : 1+ const InputIconComponent = useIcon(data.inputs.type.toLowerCase() as string, null);+ const OutputIconComponent = useIcon(data.outputs.type.toLowerCase() as string, null);+const getStatusVariant = (state?: string) => {switch (state) {@@ -92,7 +98,7 @@ const ScannerNode = memo(({ data, selected, isConnectable }: ScannerNodeProps) =<NodeStatusIndicator variant={getStatusVariant(data.computationState)}><BaseNodeSchemaclassName="shadow-md rounded-md p-0 bg-background !max-w-[340px]"- style={{ borderLeftWidth: 4, borderLeftColor: color, cursor: "grab", opacity }}+ style={{ borderLeftWidth: 8, borderRightWidth: 8, borderLeftColor: inputColor ?? outputColor, borderRightColor: outputColor, cursor: "grab", opacity }}selected={selected}><div className="p-3 bg-card rounded-t-md">@@ -114,32 +120,28 @@ const ScannerNode = memo(({ data, selected, isConnectable }: ScannerNodeProps) =<div><div className="grid grid-cols-2 py-1"><div className="pl-0 pr-6">- <p className="font-bold text-center text-xs mb-2">{data.inputs.type}</p>- {data?.inputs?.properties?.map((property, i) => (+ {data.inputs.properties.length > 0 && (<LabeledHandleisConnectable={isConnectable}- id={property.name}- name={property.name}- dataType={property.type}- key={i}+ id={data.inputs.type}+ label={<span className="flex items-center gap-1">{data.inputs.type}<InputIconComponent size={12} /></span>}+ description={`${data.inputs.properties.length} properties`}type="target"position={Position.Left}/>- ))}+ )}</div><div className="pr-0">- <p className="font-bold text-center pr-1 text-xs mb-2">{data.outputs.type}</p>- {data?.outputs?.properties?.map((property, i) => (+ {data.outputs.properties.length > 0 && (<LabeledHandleisConnectable={isConnectable}- id={property.name}- name={property.name}- dataType={property.type}- key={i}+ id={data.outputs.type}+ label={<span className="flex items-center gap-1"><OutputIconComponent size={12} />{data.outputs.type}</span>}+ description={`${data.inputs.properties.length} properties`}type="source"position={Position.Right}/>- ))}+ )}</div></div></div>diff --git a/flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsx b/flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsxindex 1f7ae022..754fc064 100644--- a/flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsx+++ b/flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsx@@ -3,11 +3,18 @@ import { UserPlus, Users } from "lucide-react"import RawMaterial from "./raw-material"import TransformsList from "./transforms-list"import { useLayoutStore } from "@/stores/layout-store"+import { useParams } from "@tanstack/react-router"+import { useEffect } from "react"const TransformNavigation = () => {+ const { transformId } = useParams({ strict: false })const activeTransformTab = useLayoutStore((s) => s.activeTransformTab)const setActiveTransformTab = useLayoutStore((s) => s.setActiveTransformTab)+ useEffect(() => {+ setActiveTransformTab(transformId ? "items" : "transforms")+ }, [setActiveTransformTab, transformId])+return (<div className="h-full w-full bg-card flex flex-col min-h-0"><Tabs@@ -17,7 +24,7 @@ const TransformNavigation = () => {className="w-full h-full flex flex-col gap-0 min-h-0"><TabsList className="w-full p-0 rounded-none my-0 border-b flex-shrink-0"><TabsTrigger value="transforms"><Users className="h-3 w-3 opacity-60" /> Transforms</TabsTrigger>- <TabsTrigger value="items"><UserPlus className="h-3 w-3 opacity-60" /> Items</TabsTrigger>+ {transformId && <TabsTrigger value="items"><UserPlus className="h-3 w-3 opacity-60" /> Items</TabsTrigger>}</TabsList><TabsContentvalue="transforms"@@ -25,12 +32,13 @@ const TransformNavigation = () => {><TransformsList /></TabsContent>- <TabsContent- value="items"- className="my-0 w-full flex-1 flex flex-col min-h-0 overflow-hidden"- >- <RawMaterial />- </TabsContent>+ {transformId &&+ <TabsContent+ value="items"+ className="my-0 w-full flex-1 flex flex-col min-h-0 overflow-hidden"+ >+ <RawMaterial />+ </TabsContent>}</Tabs></div>)diff --git a/flowsint-app/src/renderer/src/components/ui/labeled-handle.tsx b/flowsint-app/src/renderer/src/components/ui/labeled-handle.tsxindex 81a1dada..51eb8ef0 100644--- a/flowsint-app/src/renderer/src/components/ui/labeled-handle.tsx+++ b/flowsint-app/src/renderer/src/components/ui/labeled-handle.tsx@@ -1,4 +1,4 @@-import { forwardRef, type HTMLAttributes } from "react";+import { forwardRef, ReactNode, type HTMLAttributes } from "react";import { cn } from "@/lib/utils";import { type HandleProps } from "@xyflow/react";@@ -16,14 +16,14 @@ export const LabeledHandle = forwardRef<HTMLDivElement,HandleProps &HTMLAttributes<HTMLDivElement> & {- name: string | (string & Element);- dataType: string,+ label: string | (string & Element) | ReactNode;+ description: string,handleClassName?: string;labelClassName?: string;}>((- { className, labelClassName, handleClassName, name, position, dataType, ...props },+ { className, labelClassName, handleClassName, label, position, description, ...props },ref,) => (<div@@ -36,7 +36,7 @@ export const LabeledHandle = forwardRef<><BaseHandle position={position} className={handleClassName} {...props} /><label className={cn("px-3 text-foreground", labelClassName)}>- {name} <Badge variant={"outline"}>{dataType}</Badge>+ {label} <Badge variant={"outline"}>{description}</Badge></label></div>),diff --git a/flowsint-app/src/renderer/src/lib/action-items.ts b/flowsint-app/src/renderer/src/lib/action-items.tsindex 56f26d37..04ad1fe7 100644--- a/flowsint-app/src/renderer/src/lib/action-items.ts+++ b/flowsint-app/src/renderer/src/lib/action-items.ts@@ -55,7 +55,7 @@ export const actionItems: ActionItem[] = [type: "Individual",table: "individuals",key: "individual",- label_key: "first_name",+ label_key: "full_name",icon: "individual",label: "Individual",fields: [{ name: "first_name", label: "Firstname", type: "text", required: true },diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsxindex e3712ec6..a482ed56 100644--- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsx+++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsx@@ -2,8 +2,6 @@ import { createFileRoute } from '@tanstack/react-router'import { transformService } from '@/api/transfrom-service'import Editor from '@/components/transforms/editor'import Loader from '@/components/loader'-import { useEffect } from 'react'-import { useLayoutStore } from '@/stores/layout-store'export const Route = createFileRoute('/_auth/dashboard/transforms/$transformId')({loader: async ({ params: { transformId } }) => {@@ -34,12 +32,6 @@ export const Route = createFileRoute('/_auth/dashboard/transforms/$transformId')function TranformPage() {const { transform } = Route.useLoaderData()- const setActiveTransformTab = useLayoutStore((s) => s.setActiveTransformTab)-- useEffect(() => {- setActiveTransformTab("items")- }, [setActiveTransformTab])-return (<Editorkey={transform.id}diff --git a/flowsint-app/src/renderer/src/stores/node-display-settings.ts b/flowsint-app/src/renderer/src/stores/node-display-settings.tsindex e0f9d9b7..fa18bc79 100644--- a/flowsint-app/src/renderer/src/stores/node-display-settings.ts+++ b/flowsint-app/src/renderer/src/stores/node-display-settings.ts@@ -8,7 +8,7 @@ export type ItemType =| "location"| "email"| "ip"- | "social_profile"+ | "socialprofile"| "organization"| "vehicle"| "car"@@ -33,6 +33,8 @@ export type ItemType =| "siret"| "siren"| "cryptowallet"+ | "cryptotransaction"+ | "cryptonft"| "asn"| "cidr"| "whois"@@ -45,7 +47,7 @@ export const ITEM_TYPES: ItemType[] = ["location","email","ip",- "social_profile",+ "socialprofile","organization","vehicle","car",@@ -70,6 +72,8 @@ export const ITEM_TYPES: ItemType[] = ["siret","siren","cryptowallet",+ "cryptotransaction",+ "cryptonft","asn","cidr","whois",@@ -80,26 +84,26 @@ export const ITEM_TYPES: ItemType[] = [const DEFAULT_COLORS: Record<ItemType, string> = {individual: "#A9CCF4", // Notion medium bluephone: "#A0D7CF", // Notion soft teal- location: "#EAAFAF", // Notion dusty blue+ location: "#EAAFAF", // Notion dusty roseemail: "#C4B9ED", // Notion lavenderip: "#F0C19E", // Notion pale orange- social_profile: "#B8AFE6", // Notion muted purple+ socialprofile: "#D4A5D4", // Notion dusty violetorganization: "#D1C0AF", // Notion taupevehicle: "#E9CD89", // Notion wheat- car: "#E9CD89", // Notion wheat (same as vehicle)- motorcycle: "#D4C4A8", // Notion warm beige+ car: "#D4C4A8", // Notion warm beige+ motorcycle: "#C8B5A3", // Notion warm taupeboat: "#B8D4E3", // Notion soft blueplane: "#E2D1C3", // Notion light taupewebsite: "#E7B8D2", // Notion dusty rosedomain: "#A6D0BF", // Notion sagesubdomain: "#9DCBE4", // Notion sky bluedocument: "#C1C6CD", // Notion cool gray- financial: "#EAAFAF", // Notion salmon+ financial: "#F4B8A8", // Notion coralevent: "#A2D4BF", // Notion mintdevice: "#EAC597", // Notion peachmedia: "#E4B1AD", // Notion terracottaeducation: "#ABC0DA", // Notion steel blue- relationship: "#E2C3BD", // Notion dusty rose+ relationship: "#D9C2BC", // Notion dusty mauveonline_activity: "#B5D1A9", // Notion sagedigital_footprint: "#D9B0B0", // Notion brickusername: "#C7BEE4", // Notion periwinkle@@ -108,11 +112,13 @@ const DEFAULT_COLORS: Record<ItemType, string> = {siret: "#ADB9C6", // Notion blue-graysiren: "#9FAAB8", // Notion dark slatecryptowallet: "#F7D154", // Notion gold- asn: "#F7D154",- cidr: "#A2D4BF",- whois: "#EAAFAF",- gravatar: "#B8D4E3", // Notion soft blue (similar to boat)- breach: "#E4B1AD" // Notion terracotta (similar to media)+ cryptotransaction: "#E6D4A0", // Notion warm yellow+ cryptonft: "#D4E6A0", // Notion soft lime+ asn: "#E8C8A0", // Notion warm peach+ cidr: "#B8E6B8", // Notion soft mint+ whois: "#D4B5D4", // Notion dusty lavender+ gravatar: "#A8D8E8", // Notion pale blue+ breach: "#E6B8B8" // Notion soft rose}// Définition des icônes par défaut pour chaque type d'élément@@ -122,7 +128,7 @@ const DEFAULT_ICONS: Record<ItemType, string> = {location: "🏠",email: "✉️",ip: "🌐",- social_profile: "📱",+ socialprofile: "📱",organization: "🏢",vehicle: "🚗",car: "🚗",@@ -147,6 +153,8 @@ const DEFAULT_ICONS: Record<ItemType, string> = {siret: "ℹ️",siren: "ℹ️",cryptowallet: "₿",+ cryptotransaction: "💱",+ cryptonft: "🖼️",asn: "🌐",cidr: "📡",whois: "🌐",@@ -160,7 +168,7 @@ const DEFAULT_SIZES: Record<ItemType, number> = {location: 7,email: 7,ip: 7,- social_profile: 7,+ socialprofile: 7,organization: 10,vehicle: 7,car: 7,@@ -185,6 +193,8 @@ const DEFAULT_SIZES: Record<ItemType, number> = {siret: 5,siren: 5,cryptowallet: 7,+ cryptotransaction: 7,+ cryptonft: 7,asn: 12,cidr: 10,whois: 6,diff --git a/flowsint-app/src/renderer/src/stores/transform-store.ts b/flowsint-app/src/renderer/src/stores/transform-store.tsindex 5feab36e..9d3ab8bf 100644--- a/flowsint-app/src/renderer/src/stores/transform-store.ts+++ b/flowsint-app/src/renderer/src/stores/transform-store.ts@@ -11,6 +11,7 @@ import {applyNodeChanges,applyEdgeChanges} from "@xyflow/react"+import { toast } from "sonner"export interface NodeData {class_name: string@@ -71,6 +72,11 @@ export const useTransformStore = create<TransformState>((set, get) => ({})},onConnect: (connection: Connection) => {+ if (connection.sourceHandle !== connection.targetHandle) {+ toast.error(`Cannot connect ${connection.sourceHandle} to ${connection.targetHandle}.`)+ return+ }+const edge: TransformEdge = {id: `${connection.source}-${connection.target}`,source: connection.source!,