Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: progress on enricher templates

dextmorgna461f9997d08
diff --git a/flowsint-api/app/api/routes/enricher_templates.py b/flowsint-api/app/api/routes/enricher_templates.py
index fb6ed677..cf5f9479 100644
--- a/flowsint-api/app/api/routes/enricher_templates.py
+++ b/flowsint-api/app/api/routes/enricher_templates.py
@@ -12,6 +12,7 @@ from flowsint_core.core.services import (
create_enricher_template_service,
)
from flowsint_core.core.template_enricher import TemplateEnricher
+from flowsint_core.core.vault import Vault
from flowsint_core.templates.types import Template
from sqlalchemy.orm import Session
@@ -89,7 +90,11 @@ async def test_template(
try:
content = db_template.content
template = Template(**content)
- enricher = TemplateEnricher(sketch_id="123", scan_id="123", template=template)
+ vault = Vault(db=db, owner_id=current_user.id)
+ enricher = TemplateEnricher(
+ sketch_id="123", scan_id="123", template=template, vault=vault
+ )
+ await enricher.async_init()
pre = enricher.preprocess([test_request.input_value])
results = await enricher.scan(pre)
data = {"results": results}
diff --git a/flowsint-app/src/components/templates/template-editor.tsx b/flowsint-app/src/components/templates/template-editor.tsx
index be87d4c2..54c6a6b5 100644
--- a/flowsint-app/src/components/templates/template-editor.tsx
+++ b/flowsint-app/src/components/templates/template-editor.tsx
@@ -149,7 +149,8 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem
if (paramKeys.length > 0) {
const paramsObj: Record<string, string> = {}
for (const key of paramKeys) {
- const templateValue = templateParams[key] ?? ''
+ // Convert to string since YAML may parse numbers/booleans as their native types
+ const templateValue = String(templateParams[key] ?? '')
// Replace {{key}} in param value with test input
const resolvedValue = templateValue.replace(
new RegExp(`\\{\\{${inputKey}\\}\\}`, 'g'),
@@ -439,7 +440,10 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem
</div>
<div className="flex-1 flex overflow-hidden">
<Tabs value={activeTab} className="flex-1 flex">
- <TabsContent value="editor" className="flex-1 flex m-0 data-[state=inactive]:hidden">
+ <TabsContent
+ value="editor"
+ className="flex-1 grow flex m-0 data-[state=inactive]:hidden"
+ >
<div className="flex-1 min-w-0 min-h-0 overflow-hidden">
<YamlEditor
value={content}
@@ -447,7 +451,7 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem
onValidate={handleEditorValidate}
/>
</div>
- <div className="w-72 border-l bg-card/30 flex flex-col">
+ <div className="w-72 border-l bg-card/30 flex flex-col overflow-auto">
<div className="p-4 border-b">
<div className="flex items-center gap-2">
{hasErrors ? (
@@ -479,7 +483,7 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem
)}
</div>
</div>
- <ScrollArea className="flex-1">
+ <div className="flex-1 grow overflow-auto">
<div className="p-4 space-y-3">
{validationResult.errors.map((error, i) => (
<div key={i} className="flex gap-2 text-sm">
@@ -529,7 +533,7 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem
</div>
</div>
</div>
- </ScrollArea>
+ </div>
</div>
</TabsContent>
diff --git a/flowsint-app/src/components/templates/template-schema.ts b/flowsint-app/src/components/templates/template-schema.ts
index 59be4829..0c171daf 100644
--- a/flowsint-app/src/components/templates/template-schema.ts
+++ b/flowsint-app/src/components/templates/template-schema.ts
@@ -172,23 +172,90 @@ export const templateSchema = {
}
}
-export const defaultTemplate = `name: my-enricher
-description: My custom enricher template
-category: Domain
-type: request
+export const defaultTemplate = `# Example template enricher
+# GitHub user lookup template
+# Fetches user profile information from GitHub API
+#
+# API endpoint: https://api.github.com/users/{username}
+# Docs: https://docs.github.com/en/rest/users/users#get-a-user
+#
+# Example response:
+# {
+# "login": "my_gh_pseudo",
+# "id": 206358,
+# "avatar_url": "https://avatars.githubusercontent.com/u/206358?v=4",
+# "html_url": "https://github.com/my_gh_pseudo",
+# "name": "John Doe",
+# "bio": "Developer",
+# "location": "San Francisco",
+# "followers": 3,
+# "following": 0,
+# "public_repos": 1,
+# "created_at": "2010-02-18T23:00:25Z",
+# ...
+# }
+
+name: github-user-lookup
+description: Fetch GitHub user profile and return as SocialAccount
+category: Username
version: 1.0
+
input:
- type: Domain
- key: domain
+ type: Username
+ key: value
+
+secrets:
+ - name: GITHUB_TOKEN
+ required: true
+ description: GitHub personal access token (required for API rate limits)
+
request:
method: GET
- url: https://api.example.com/lookup/{{domain}}
-output:
- type: Domain
+ url: https://api.github.com/users/{{value}}
+ headers:
+ Accept: application/vnd.github+json
+ Authorization: Bearer {{secrets.GITHUB_TOKEN}}
+ X-GitHub-Api-Version: "2022-11-28"
+ User-Agent: flowsint-enricher
+ timeout: 15
+
response:
expect: json
map:
- domain: domain
+ # SocialAccount.username <- response["login"]
+ username: login
+ # SocialAccount.display_name <- response["name"]
+ display_name: name
+ # SocialAccount.profile_url <- response["html_url"]
+ profile_url: html_url
+ # SocialAccount.profile_picture_url <- response["avatar_url"]
+ profile_picture_url: avatar_url
+ # SocialAccount.bio <- response["bio"]
+ bio: bio
+ # SocialAccount.location <- response["location"]
+ location: location
+ # SocialAccount.created_at <- response["created_at"]
+ created_at: created_at
+ # SocialAccount.followers_count <- response["followers"]
+ followers_count: followers
+ # SocialAccount.following_count <- response["following"]
+ following_count: following
+ # SocialAccount.posts_count <- response["public_repos"]
+ posts_count: public_repos
+
+output:
+ type: SocialAccount
+
+retry:
+ max_retries: 3
+ backoff_factor: 1.0
+ retry_on_status:
+ - 429
+ - 500
+ - 502
+ - 503
+ - 504
+
`
export interface TemplateInput {
diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.py
index 279fe119..430f6978 100644
--- a/flowsint-core/src/flowsint_core/core/template_enricher.py
+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py
@@ -517,15 +517,15 @@ class TemplateEnricher(Enricher):
except Exception as e:
Logger.info(
self.sketch_id,
- {"message": f"Unexpected error processing {self.request.url}: {e}"},
+ {
+ "message": f"Unexpected error processing {self.request.url}: {e}"
+ },
)
continue
return results
- def postprocess(
- self, results: List[Any], input_data: List[Any] = []
- ) -> List[Any]:
+ def postprocess(self, results: List[Any], input_data: List[Any] = []) -> List[Any]:
"""Log results and return them."""
for res in results:
self.log_graph_message(
diff --git a/flowsint-core/tests/templates/example-github-user.yaml b/flowsint-core/tests/templates/example-github-user.yaml
index 8b552e90..fc90c665 100644
--- a/flowsint-core/tests/templates/example-github-user.yaml
+++ b/flowsint-core/tests/templates/example-github-user.yaml
@@ -1,82 +1,43 @@
-# GitHub User Lookup Template
-# Fetches user profile information from GitHub API
-#
-# API endpoint: https://api.github.com/users/{username}
-# Docs: https://docs.github.com/en/rest/users/users#get-a-user
-#
-# Example response:
-# {
-# "login": "testee",
-# "id": 206358,
-# "avatar_url": "https://avatars.githubusercontent.com/u/206358?v=4",
-# "html_url": "https://github.com/testee",
-# "name": "Test User",
-# "bio": "Developer",
-# "location": "San Francisco",
-# "followers": 3,
-# "following": 0,
-# "public_repos": 1,
-# "created_at": "2010-02-18T23:00:25Z",
-# ...
-# }
-
name: github-user-lookup
-description: Fetch GitHub user profile and return as SocialAccount
-category: Username
-version: 1.0
-
input:
- type: Username
key: value
+ type: Username
+retry:
+ max_retries: 3
+ backoff_factor: 1
+ retry_on_status:
+ - 429
+ - 500
+ - 502
+ - 503
+ - 504
+output:
+ type: SocialAccount
+request:
+ url: https://api.github.com/users/{{value}}
+ params:
+ per_page: 100
+ sort: updated
+ method: GET
+ timeout: 15
secrets:
- name: GITHUB_TOKEN
required: true
description: GitHub personal access token (required for API rate limits)
-
-request:
- method: GET
- url: https://api.github.com/users/{{value}}
- headers:
- Accept: application/vnd.github+json
- Authorization: Bearer {{secrets.GITHUB_TOKEN}}
- X-GitHub-Api-Version: "2022-11-28"
- User-Agent: flowsint-enricher
- timeout: 15
-
+version: 1
+category: Username
response:
- expect: json
map:
- # SocialAccount.username <- response["login"]
- username: login
- # SocialAccount.display_name <- response["name"]
- display_name: name
- # SocialAccount.profile_url <- response["html_url"]
- profile_url: html_url
- # SocialAccount.profile_picture_url <- response["avatar_url"]
- profile_picture_url: avatar_url
- # SocialAccount.bio <- response["bio"]
bio: bio
- # SocialAccount.location <- response["location"]
location: location
- # SocialAccount.created_at <- response["created_at"]
+ username: login
created_at: created_at
- # SocialAccount.followers_count <- response["followers"]
+ posts_count: public_repos
+ profile_url: html_url
+ display_name: name
followers_count: followers
- # SocialAccount.following_count <- response["following"]
following_count: following
- # SocialAccount.posts_count <- response["public_repos"]
- posts_count: public_repos
-
-output:
- type: SocialAccount
-
-retry:
- max_retries: 3
- backoff_factor: 1.0
- retry_on_status:
- - 429
- - 500
- - 502
- - 503
- - 504
+ profile_picture_url: avatar_url
+ expect: json
+description: Fetch GitHub user profile and return as SocialAccount
diff --git a/flowsint-core/tests/templates/example-github-v2.yaml b/flowsint-core/tests/templates/example-github-v2.yaml
new file mode 100644
index 00000000..2035fb09
--- /dev/null
+++ b/flowsint-core/tests/templates/example-github-v2.yaml
@@ -0,0 +1,71 @@
+name: github-user-lookup
+description: Fetch GitHub user profile and repos; emit SocialAccount + Posts
+category: Username
+version: 1.1
+
+input:
+ type: Username
+ key: value
+
+secrets:
+ - name: GITHUB_TOKEN
+ required: true
+
+request:
+ - id: user
+ method: GET
+ url: https://api.github.com/users/{{value}}
+ headers:
+ Accept: application/vnd.github+json
+ Authorization: Bearer {{secrets.GITHUB_TOKEN}}
+ X-GitHub-Api-Version: "2022-11-28"
+ User-Agent: flowsint-enricher
+ timeout: 15
+
+ - id: repos
+ method: GET
+ url: https://api.github.com/users/{{value}}/repos?per_page=100&sort=updated
+ headers:
+ Accept: application/vnd.github+json
+ Authorization: Bearer {{secrets.GITHUB_TOKEN}}
+ X-GitHub-Api-Version: "2022-11-28"
+ User-Agent: flowsint-enricher
+ timeout: 15
+
+response:
+ expect: json
+
+outputs:
+ - type: SocialAccount
+ from: user
+ map:
+ username: login
+ display_name: name
+ profile_url: html_url
+ profile_picture_url: avatar_url
+ bio: bio
+ location: location
+ created_at: created_at
+ followers_count: followers
+ following_count: following
+ posts_count: public_repos
+
+ - type: Post
+ from: repos
+ for_each: "$" # iterate over the root array
+ map:
+ id: id
+ title: name
+ url: html_url
+ created_at: created_at
+ updated_at: updated_at
+ description: description
+ language: language
+ stars: stargazers_count
+ owner_username: owner.login
+ owner_profile_url: owner.html_url
+
+retry:
+ max_retries: 3
+ backoff_factor: 1.0
+ retry_on_status: [429, 500, 502, 503, 504]