A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: progress on enricher templates
dextmorgna461f9997d08
diff --git a/flowsint-api/app/api/routes/enricher_templates.py b/flowsint-api/app/api/routes/enricher_templates.pyindex fb6ed677..cf5f9479 100644--- a/flowsint-api/app/api/routes/enricher_templates.py+++ b/flowsint-api/app/api/routes/enricher_templates.py@@ -12,6 +12,7 @@ from flowsint_core.core.services import (create_enricher_template_service,)from flowsint_core.core.template_enricher import TemplateEnricher+from flowsint_core.core.vault import Vaultfrom flowsint_core.templates.types import Templatefrom sqlalchemy.orm import Session@@ -89,7 +90,11 @@ async def test_template(try:content = db_template.contenttemplate = Template(**content)- enricher = TemplateEnricher(sketch_id="123", scan_id="123", template=template)+ vault = Vault(db=db, owner_id=current_user.id)+ enricher = TemplateEnricher(+ sketch_id="123", scan_id="123", template=template, vault=vault+ )+ await enricher.async_init()pre = enricher.preprocess([test_request.input_value])results = await enricher.scan(pre)data = {"results": results}diff --git a/flowsint-app/src/components/templates/template-editor.tsx b/flowsint-app/src/components/templates/template-editor.tsxindex be87d4c2..54c6a6b5 100644--- a/flowsint-app/src/components/templates/template-editor.tsx+++ b/flowsint-app/src/components/templates/template-editor.tsx@@ -149,7 +149,8 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Temif (paramKeys.length > 0) {const paramsObj: Record<string, string> = {}for (const key of paramKeys) {- const templateValue = templateParams[key] ?? ''+ // Convert to string since YAML may parse numbers/booleans as their native types+ const templateValue = String(templateParams[key] ?? '')// Replace {{key}} in param value with test inputconst resolvedValue = templateValue.replace(new RegExp(`\\{\\{${inputKey}\\}\\}`, 'g'),@@ -439,7 +440,10 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem</div><div className="flex-1 flex overflow-hidden"><Tabs value={activeTab} className="flex-1 flex">- <TabsContent value="editor" className="flex-1 flex m-0 data-[state=inactive]:hidden">+ <TabsContent+ value="editor"+ className="flex-1 grow flex m-0 data-[state=inactive]:hidden"+ ><div className="flex-1 min-w-0 min-h-0 overflow-hidden"><YamlEditorvalue={content}@@ -447,7 +451,7 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: TemonValidate={handleEditorValidate}/></div>- <div className="w-72 border-l bg-card/30 flex flex-col">+ <div className="w-72 border-l bg-card/30 flex flex-col overflow-auto"><div className="p-4 border-b"><div className="flex items-center gap-2">{hasErrors ? (@@ -479,7 +483,7 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem)}</div></div>- <ScrollArea className="flex-1">+ <div className="flex-1 grow overflow-auto"><div className="p-4 space-y-3">{validationResult.errors.map((error, i) => (<div key={i} className="flex gap-2 text-sm">@@ -529,7 +533,7 @@ export function TemplateEditor({ templateId, initialContent, importedYaml }: Tem</div></div></div>- </ScrollArea>+ </div></div></TabsContent>diff --git a/flowsint-app/src/components/templates/template-schema.ts b/flowsint-app/src/components/templates/template-schema.tsindex 59be4829..0c171daf 100644--- a/flowsint-app/src/components/templates/template-schema.ts+++ b/flowsint-app/src/components/templates/template-schema.ts@@ -172,23 +172,90 @@ export const templateSchema = {}}-export const defaultTemplate = `name: my-enricher-description: My custom enricher template-category: Domain-type: request+export const defaultTemplate = `# Example template enricher+# GitHub user lookup template+# Fetches user profile information from GitHub API+#+# API endpoint: https://api.github.com/users/{username}+# Docs: https://docs.github.com/en/rest/users/users#get-a-user+#+# Example response:+# {+# "login": "my_gh_pseudo",+# "id": 206358,+# "avatar_url": "https://avatars.githubusercontent.com/u/206358?v=4",+# "html_url": "https://github.com/my_gh_pseudo",+# "name": "John Doe",+# "bio": "Developer",+# "location": "San Francisco",+# "followers": 3,+# "following": 0,+# "public_repos": 1,+# "created_at": "2010-02-18T23:00:25Z",+# ...+# }++name: github-user-lookup+description: Fetch GitHub user profile and return as SocialAccount+category: Usernameversion: 1.0+input:- type: Domain- key: domain+ type: Username+ key: value++secrets:+ - name: GITHUB_TOKEN+ required: true+ description: GitHub personal access token (required for API rate limits)+request:method: GET- url: https://api.example.com/lookup/{{domain}}-output:- type: Domain+ url: https://api.github.com/users/{{value}}+ headers:+ Accept: application/vnd.github+json+ Authorization: Bearer {{secrets.GITHUB_TOKEN}}+ X-GitHub-Api-Version: "2022-11-28"+ User-Agent: flowsint-enricher+ timeout: 15+response:expect: jsonmap:- domain: domain+ # SocialAccount.username <- response["login"]+ username: login+ # SocialAccount.display_name <- response["name"]+ display_name: name+ # SocialAccount.profile_url <- response["html_url"]+ profile_url: html_url+ # SocialAccount.profile_picture_url <- response["avatar_url"]+ profile_picture_url: avatar_url+ # SocialAccount.bio <- response["bio"]+ bio: bio+ # SocialAccount.location <- response["location"]+ location: location+ # SocialAccount.created_at <- response["created_at"]+ created_at: created_at+ # SocialAccount.followers_count <- response["followers"]+ followers_count: followers+ # SocialAccount.following_count <- response["following"]+ following_count: following+ # SocialAccount.posts_count <- response["public_repos"]+ posts_count: public_repos++output:+ type: SocialAccount++retry:+ max_retries: 3+ backoff_factor: 1.0+ retry_on_status:+ - 429+ - 500+ - 502+ - 503+ - 504+`export interface TemplateInput {diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.pyindex 279fe119..430f6978 100644--- a/flowsint-core/src/flowsint_core/core/template_enricher.py+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py@@ -517,15 +517,15 @@ class TemplateEnricher(Enricher):except Exception as e:Logger.info(self.sketch_id,- {"message": f"Unexpected error processing {self.request.url}: {e}"},+ {+ "message": f"Unexpected error processing {self.request.url}: {e}"+ },)continuereturn results- def postprocess(- self, results: List[Any], input_data: List[Any] = []- ) -> List[Any]:+ def postprocess(self, results: List[Any], input_data: List[Any] = []) -> List[Any]:"""Log results and return them."""for res in results:self.log_graph_message(diff --git a/flowsint-core/tests/templates/example-github-user.yaml b/flowsint-core/tests/templates/example-github-user.yamlindex 8b552e90..fc90c665 100644--- a/flowsint-core/tests/templates/example-github-user.yaml+++ b/flowsint-core/tests/templates/example-github-user.yaml@@ -1,82 +1,43 @@-# GitHub User Lookup Template-# Fetches user profile information from GitHub API-#-# API endpoint: https://api.github.com/users/{username}-# Docs: https://docs.github.com/en/rest/users/users#get-a-user-#-# Example response:-# {-# "login": "testee",-# "id": 206358,-# "avatar_url": "https://avatars.githubusercontent.com/u/206358?v=4",-# "html_url": "https://github.com/testee",-# "name": "Test User",-# "bio": "Developer",-# "location": "San Francisco",-# "followers": 3,-# "following": 0,-# "public_repos": 1,-# "created_at": "2010-02-18T23:00:25Z",-# ...-# }-name: github-user-lookup-description: Fetch GitHub user profile and return as SocialAccount-category: Username-version: 1.0-input:- type: Usernamekey: value+ type: Username+retry:+ max_retries: 3+ backoff_factor: 1+ retry_on_status:+ - 429+ - 500+ - 502+ - 503+ - 504+output:+ type: SocialAccount+request:+ url: https://api.github.com/users/{{value}}+ params:+ per_page: 100+ sort: updated+ method: GET+ timeout: 15secrets:- name: GITHUB_TOKENrequired: truedescription: GitHub personal access token (required for API rate limits)--request:- method: GET- url: https://api.github.com/users/{{value}}- headers:- Accept: application/vnd.github+json- Authorization: Bearer {{secrets.GITHUB_TOKEN}}- X-GitHub-Api-Version: "2022-11-28"- User-Agent: flowsint-enricher- timeout: 15-+version: 1+category: Usernameresponse:- expect: jsonmap:- # SocialAccount.username <- response["login"]- username: login- # SocialAccount.display_name <- response["name"]- display_name: name- # SocialAccount.profile_url <- response["html_url"]- profile_url: html_url- # SocialAccount.profile_picture_url <- response["avatar_url"]- profile_picture_url: avatar_url- # SocialAccount.bio <- response["bio"]bio: bio- # SocialAccount.location <- response["location"]location: location- # SocialAccount.created_at <- response["created_at"]+ username: logincreated_at: created_at- # SocialAccount.followers_count <- response["followers"]+ posts_count: public_repos+ profile_url: html_url+ display_name: namefollowers_count: followers- # SocialAccount.following_count <- response["following"]following_count: following- # SocialAccount.posts_count <- response["public_repos"]- posts_count: public_repos--output:- type: SocialAccount--retry:- max_retries: 3- backoff_factor: 1.0- retry_on_status:- - 429- - 500- - 502- - 503- - 504+ profile_picture_url: avatar_url+ expect: json+description: Fetch GitHub user profile and return as SocialAccountdiff --git a/flowsint-core/tests/templates/example-github-v2.yaml b/flowsint-core/tests/templates/example-github-v2.yamlnew file mode 100644index 00000000..2035fb09--- /dev/null+++ b/flowsint-core/tests/templates/example-github-v2.yaml@@ -0,0 +1,71 @@+name: github-user-lookup+description: Fetch GitHub user profile and repos; emit SocialAccount + Posts+category: Username+version: 1.1++input:+ type: Username+ key: value++secrets:+ - name: GITHUB_TOKEN+ required: true++request:+ - id: user+ method: GET+ url: https://api.github.com/users/{{value}}+ headers:+ Accept: application/vnd.github+json+ Authorization: Bearer {{secrets.GITHUB_TOKEN}}+ X-GitHub-Api-Version: "2022-11-28"+ User-Agent: flowsint-enricher+ timeout: 15++ - id: repos+ method: GET+ url: https://api.github.com/users/{{value}}/repos?per_page=100&sort=updated+ headers:+ Accept: application/vnd.github+json+ Authorization: Bearer {{secrets.GITHUB_TOKEN}}+ X-GitHub-Api-Version: "2022-11-28"+ User-Agent: flowsint-enricher+ timeout: 15++response:+ expect: json++outputs:+ - type: SocialAccount+ from: user+ map:+ username: login+ display_name: name+ profile_url: html_url+ profile_picture_url: avatar_url+ bio: bio+ location: location+ created_at: created_at+ followers_count: followers+ following_count: following+ posts_count: public_repos++ - type: Post+ from: repos+ for_each: "$" # iterate over the root array+ map:+ id: id+ title: name+ url: html_url+ created_at: created_at+ updated_at: updated_at+ description: description+ language: language+ stars: stargazers_count+ owner_username: owner.login+ owner_profile_url: owner.html_url++retry:+ max_retries: 3+ backoff_factor: 1.0+ retry_on_status: [429, 500, 502, 503, 504]