Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(siem): constrain audit metadata to username (#4144)
Gergő Móricz1a55ed63a91f
diff --git a/apps/api/src/controllers/v2/siem-logging-schema.test.ts b/apps/api/src/controllers/v2/siem-logging-schema.test.tsindex cc5fe808e..66fb606eb 100644--- a/apps/api/src/controllers/v2/siem-logging-schema.test.ts+++ b/apps/api/src/controllers/v2/siem-logging-schema.test.ts@@ -20,7 +20,6 @@ import { siemLoggingConfigInputSchema } from "../../lib/siem-logging/types";const auditMetadata = {username: "[email protected]",- session: "session-123",};describe("auditMetadata request validation", () => {@@ -69,16 +68,25 @@ describe("auditMetadata request validation", () => {).toEqual(auditMetadata);});- it("rejects oversized metadata", () => {- const tooManyFields = Object.fromEntries(- Array.from({ length: 33 }, (_, index) => [`field-${index}`, "value"]),- );+ it("rejects unsupported audit metadata fields", () => {expect(() =>scrapeRequestSchema.parse({url: "https://example.com",- auditMetadata: tooManyFields,+ auditMetadata: {+ username: "[email protected]",+ session: "session-123",+ },}),- ).toThrow("at most 32 fields");+ ).toThrow();+ });++ it("requires username when audit metadata is provided", () => {+ expect(() =>+ scrapeRequestSchema.parse({+ url: "https://example.com",+ auditMetadata: {},+ }),+ ).toThrow();});it("accepts audit metadata on the corresponding v1 request schemas", () => {diff --git a/apps/api/src/controllers/v2/team-siem-logging.ts b/apps/api/src/controllers/v2/team-siem-logging.tsindex b69a141e2..f31a11635 100644--- a/apps/api/src/controllers/v2/team-siem-logging.ts+++ b/apps/api/src/controllers/v2/team-siem-logging.ts@@ -137,7 +137,7 @@ export async function testTeamSiemLoggingController(id: req.acuc?.api_key_id == null ? null : String(req.acuc.api_key_id),name: null,},- audit_metadata: { synthetic: "true" },+ audit_metadata: { username: "siem-test" },started_at: now,completed_at: now,url: "https://example.com/siem-test",diff --git a/apps/api/src/lib/siem-logging/event.test.ts b/apps/api/src/lib/siem-logging/event.test.tsindex 55d622597..373de6081 100644--- a/apps/api/src/lib/siem-logging/event.test.ts+++ b/apps/api/src/lib/siem-logging/event.test.ts@@ -85,6 +85,22 @@ describe("ScrapeActivityEvent", () => {expect(JSON.stringify(event)).not.toContain("not included");});+ it("omits audit metadata when no username is provided", () => {+ const event = buildScrapeActivityEvent(+ "scrape-id",+ job({ scrapeOptions: scrapeOptions.parse({}) }),+ "org-id",+ null,+ {+ success: true,+ startedAt: 1_000,+ completedAt: 2_000,+ },+ );++ expect(event).not.toHaveProperty("audit_metadata");+ });+it("normalizes a blocked threat decision and error", () => {const denied: ThreatDecision = {...allowedDecision,diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.tsindex 8fa713dcd..4651ee7de 100644--- a/apps/api/src/lib/siem-logging/event.ts+++ b/apps/api/src/lib/siem-logging/event.ts@@ -171,7 +171,6 @@ function buildEvent(id: source.apiKeyId == null ? null : String(source.apiKeyId),name: apiKeyName,},- audit_metadata: source.auditMetadata ?? {},started_at: new Date(outcome.startedAt).toISOString(),completed_at: new Date(outcome.completedAt).toISOString(),url: source.url,@@ -184,6 +183,7 @@ function buildEvent(integration: source.integration ?? null,zero_data_retention: source.zeroDataRetention,};+ if (source.auditMetadata) event.audit_metadata = source.auditMetadata;const threat = threatForDecisions(outcome.threatDecisions);if (threat) event.threat = threat;return event;diff --git a/apps/api/src/lib/siem-logging/queue.test.ts b/apps/api/src/lib/siem-logging/queue.test.tsindex 6fe00ee56..533c7ea61 100644--- a/apps/api/src/lib/siem-logging/queue.test.ts+++ b/apps/api/src/lib/siem-logging/queue.test.ts@@ -25,7 +25,6 @@ const event: ScrapeActivityEvent = {team_id: "team-id",org_id: "org-id",api_key: { id: null, name: null },- audit_metadata: {},started_at: "2026-07-27T00:00:00.000Z",completed_at: "2026-07-27T00:00:01.000Z",url: "https://example.com",diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.tsindex c6f4b3af1..efb2c91ec 100644--- a/apps/api/src/lib/siem-logging/types.ts+++ b/apps/api/src/lib/siem-logging/types.ts@@ -4,11 +4,9 @@ import type {ThreatProvider,} from "../threat-protection/types";-export const auditMetadataSchema = z- .record(z.string().min(1).max(64), z.string().max(1024))- .refine(value => Object.keys(value).length <= 32, {- message: "auditMetadata may contain at most 32 fields",- });+export const auditMetadataSchema = z.strictObject({+ username: z.string().max(1024),+});export type AuditMetadata = z.infer<typeof auditMetadataSchema>;@@ -107,7 +105,7 @@ export interface ScrapeActivityEvent {id: string | null;name: string | null;};- audit_metadata: AuditMetadata;+ audit_metadata?: AuditMetadata;started_at: string;completed_at: string;url: string;diff --git a/apps/api/src/services/siem-logging/worker.test.ts b/apps/api/src/services/siem-logging/worker.test.tsindex e83e2a57b..4e8fac631 100644--- a/apps/api/src/services/siem-logging/worker.test.ts+++ b/apps/api/src/services/siem-logging/worker.test.ts@@ -17,7 +17,6 @@ const event: ScrapeActivityEvent = {team_id: "team-id",org_id: "org-id",api_key: { id: null, name: null },- audit_metadata: {},started_at: "2026-07-27T00:00:00.000Z",completed_at: "2026-07-27T00:00:01.000Z",url: "https://example.com",diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.csindex 3d80e6503..3f95992ca 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs@@ -18,7 +18,7 @@ public class ModelsTestsTimeout = 30000,Mobile = false,RedactPII = true,- AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" }+ AuditMetadata = new AuditMetadata { Username = "[email protected]" }};var json = JsonSerializer.Serialize(options, JsonOptions);@@ -29,7 +29,7 @@ public class ModelsTestsAssert.Contains("\"timeout\":30000", json);Assert.Contains("\"mobile\":false", json);Assert.Contains("\"redactPII\":true", json);- Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);+ Assert.Contains("\"auditMetadata\":{\"username\":\"[email protected]\"}", json);}[Fact]@@ -73,14 +73,14 @@ public class ModelsTestsSearch = "pricing",Limit = 10,IncludeSubdomains = true,- AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" }+ AuditMetadata = new AuditMetadata { Username = "[email protected]" }};var json = JsonSerializer.Serialize(options, JsonOptions);Assert.Contains("\"search\":\"pricing\"", json);Assert.Contains("\"limit\":10", json);Assert.Contains("\"includeSubdomains\":true", json);- Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);+ Assert.Contains("\"auditMetadata\":{\"username\":\"[email protected]\"}", json);}[Fact]diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.csindex 8708c2b86..f49544c53 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs@@ -57,7 +57,7 @@ public class ParseTestsTimeout = 30000,Proxy = "auto",RedactPII = true,- AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" },+ AuditMetadata = new AuditMetadata { Username = "[email protected]" },};var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);@@ -66,7 +66,7 @@ public class ParseTestsAssert.Contains("\"timeout\":30000", json);Assert.Contains("\"proxy\":\"auto\"", json);Assert.Contains("\"redactPII\":true", json);- Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);+ Assert.Contains("\"auditMetadata\":{\"username\":\"[email protected]\"}", json);}[Fact]diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex 56c8588de..01f2feea9 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.10.0</Version>+ <Version>1.10.1</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex 0fa4f9729..52c02acc2 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -714,7 +714,7 @@ public class FirecrawlClient// INTERNAL UTILITIES// ================================================================- private const string SdkOrigin = "[email protected]";+ private const string SdkOrigin = "[email protected]";private static Dictionary<string, object> BuildBody(object? options){diff --git a/apps/dot-net-sdk/Firecrawl/Models/AuditMetadata.cs b/apps/dot-net-sdk/Firecrawl/Models/AuditMetadata.csnew file mode 100644index 000000000..3638b4c46--- /dev/null+++ b/apps/dot-net-sdk/Firecrawl/Models/AuditMetadata.cs@@ -0,0 +1,9 @@+using System.Text.Json.Serialization;++namespace Firecrawl.Models;++public sealed class AuditMetadata+{+ [JsonPropertyName("username")]+ public required string Username { get; set; }+}diff --git a/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/MapOptions.csindex 8b33ec39c..8268f0586 100644--- a/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs@@ -45,5 +45,5 @@ public class MapOptions[JsonPropertyName("auditMetadata")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]- public Dictionary<string, string>? AuditMetadata { get; set; }+ public AuditMetadata? AuditMetadata { get; set; }}diff --git a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.csindex b1c92bc6f..1992811ff 100644--- a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs@@ -85,7 +85,7 @@ public class ParseOptions[JsonPropertyName("auditMetadata")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]- public Dictionary<string, string>? AuditMetadata { get; set; }+ public AuditMetadata? AuditMetadata { get; set; }/// <summary>/// Validate the options against /v2/parse's supported surface.diff --git a/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.csindex 4496de609..8a7b561c1 100644--- a/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs@@ -81,7 +81,7 @@ public class ScrapeOptions[JsonPropertyName("auditMetadata")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]- public Dictionary<string, string>? AuditMetadata { get; set; }+ public AuditMetadata? AuditMetadata { get; set; }[JsonPropertyName("integration")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exsindex 54ba3f095..c826dde40 100644--- a/apps/elixir-sdk/generate.exs+++ b/apps/elixir-sdk/generate.exs@@ -145,7 +145,7 @@ defmodule Firecrawl.Generator do@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}@base_url "#{base_url}"- @sdk_origin "[email protected]"+ @sdk_origin "[email protected]"defp client(opts) doapi_key =@@ -663,11 +663,10 @@ defmodule Firecrawl.Generator doproperties|> Enum.map(fn {name, prop_schema} ->snake = to_snake_case(name)- type = openapi_to_nimble_type(prop_schema)required = name in required_keysdoc = Map.get(prop_schema, "description", "")- parts = ["type: #{type}"]+ parts = openapi_to_nimble_parts(name, prop_schema)parts = if required, do: parts ++ ["required: true"], else: partsparts = if doc != "", do: parts ++ ["doc: #{inspect(doc)}"], else: parts@@ -733,6 +732,28 @@ defmodule Firecrawl.Generator do# OpenAPI → NimbleOptions type mapping# ---------------------------------------------------------------------------+ defp openapi_to_nimble_parts(+ "auditMetadata",+ %{"type" => "object", "properties" => properties} = schema+ ) do+ required = Map.get(schema, "required", [])++ keys =+ properties+ |> Enum.map(fn {name, property_schema} ->+ parts = ["type: #{openapi_to_nimble_type(property_schema)}"]+ parts = if name in required, do: parts ++ ["required: true"], else: parts+ "#{to_snake_case(name)}: [#{Enum.join(parts, ", ")}]"+ end)+ |> Enum.join(", ")++ ["type: :keyword_list", "keys: [#{keys}]"]+ end++ defp openapi_to_nimble_parts(_name, schema) do+ ["type: #{openapi_to_nimble_type(schema)}"]+ end+defp openapi_to_nimble_type(%{"type" => "string", "enum" => values}) doinspected = values |> Enum.map(&atom_literal/1) |> Enum.join(", ")"{:or, [{:in, [#{inspected}]}, :string]}"diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex e390b6cc9..885262524 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -41,7 +41,7 @@ defmodule Firecrawl do@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}@base_url "https://api.firecrawl.dev/v2"- @sdk_origin "[email protected]"+ @sdk_origin "[email protected]"defp client(opts) doapi_key =@@ -867,7 +867,7 @@ defmodule Firecrawl do@map_urls_schema NimbleOptions.new!([- audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],+ audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution to include with SIEM logging events."],ignore_cache: [type: :boolean, doc: "Bypass the sitemap cache to retrieve fresh URLs. Sitemap data is cached for up to 7 days; use this parameter when your sitemap has been recently updated."],ignore_query_parameters: [type: :boolean, doc: "Do not return URLs with query parameters"],include_subdomains: [type: :boolean, doc: "Include subdomains of the website"],@@ -917,7 +917,7 @@ defmodule Firecrawl do@parse_file_schema NimbleOptions.new!([- audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],+ audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution to include with SIEM logging events."],block_ads: [type: :boolean, doc: "Enable ad and cookie popup blocking."],exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],formats: [type: {:list, :any}, doc: "Output formats supported for `/parse` uploads. Browser-rendering formats and change tracking are not supported."],@@ -1019,7 +1019,7 @@ defmodule Firecrawl do@scrape_and_extract_from_url_schema NimbleOptions.new!([url: [type: :string, required: true, doc: "The URL to scrape"],- audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],+ audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution to include with SIEM logging events."],actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],@@ -1082,7 +1082,7 @@ defmodule Firecrawl do@scrape_and_extract_from_urls_schema NimbleOptions.new!([- audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],+ audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution to include with SIEM logging events."],ignore_invalid_urls: [type: :boolean, doc: "If invalid URLs are specified in the urls array, they will be ignored. Instead of them failing the entire request, a batch scrape using the remaining valid URLs will be created, and the invalid URLs will be returned in the invalidURLs field of the response."],max_concurrency: [type: :integer, doc: "Maximum number of concurrent scrapes. This parameter allows you to set a concurrency limit for this batch scrape. If not specified, the batch scrape adheres to your team's concurrency limit."],urls: [type: {:list, :string}, required: true],@@ -1346,7 +1346,7 @@ defmodule Firecrawl do@start_agent_schema NimbleOptions.new!([- audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],+ audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution to include with SIEM logging events."],max_credits: [type: {:or, [:integer, :float]}, doc: "Maximum credits to spend on this agent task. Defaults to 2500 if not set. Values above 2,500 are always billed as paid requests."],model: [type: {:or, [{:in, [:"spark-1-mini", :"spark-1-pro"]}, :string]}, doc: "The model to use for the agent task. spark-1-mini (default) is 60% cheaper, spark-1-pro offers higher accuracy for complex tasks"],prompt: [type: :string, required: true, doc: "The prompt describing what data to extract"],diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 5013beee7..a9fe4235a 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.9.0"+ @version "1.9.1"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exsindex 4dcc3f4d3..89e70c556 100644--- a/apps/elixir-sdk/test/firecrawl_test.exs+++ b/apps/elixir-sdk/test/firecrawl_test.exs@@ -332,7 +332,8 @@ defmodule FirecrawlTest do{request, resp}end- metadata = %{"requestId" => "req-123"}+ metadata = [username: "[email protected]"]+ serialized_metadata = %{"username" => "[email protected]"}requests = [fn ->@@ -369,10 +370,22 @@ defmodule FirecrawlTest dotrue -> request.options[:json]end- assert body["auditMetadata"] == metadata+ assert body["auditMetadata"] == serialized_metadataend)end+ test "audit_metadata rejects unsupported fields" do+ assert_raise NimbleOptions.ValidationError, fn ->+ Firecrawl.scrape_and_extract_from_url(+ [+ url: "https://example.com",+ audit_metadata: [username: "[email protected]", session: "session-123"]+ ],+ api_key: "test-key"+ )+ end+ end+test "search maps highlights to highlights" doparent = self()diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.goindex 9cb22232f..6fa23d7e5 100644--- a/apps/go-sdk/options.go+++ b/apps/go-sdk/options.go@@ -91,7 +91,7 @@ type ScrapeOptions struct {StoreInCache *bool `json:"storeInCache,omitempty"`Lockdown *bool `json:"lockdown,omitempty"`RedactPII *bool `json:"redactPII,omitempty"`- AuditMetadata map[string]string `json:"auditMetadata,omitempty"`+ AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`Integration *string `json:"integration,omitempty"`JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`}@@ -151,15 +151,15 @@ type BatchScrapeOptions struct {// MapOptions configures a map (URL discovery) request.type MapOptions struct {- Search *string `json:"search,omitempty"`- Sitemap *string `json:"sitemap,omitempty"`- IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`- IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`- Limit *int `json:"limit,omitempty"`- Timeout *int `json:"timeout,omitempty"`- Integration *string `json:"integration,omitempty"`- Location *LocationConfig `json:"location,omitempty"`- AuditMetadata map[string]string `json:"auditMetadata,omitempty"`+ Search *string `json:"search,omitempty"`+ Sitemap *string `json:"sitemap,omitempty"`+ IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`+ IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`+ Limit *int `json:"limit,omitempty"`+ Timeout *int `json:"timeout,omitempty"`+ Integration *string `json:"integration,omitempty"`+ Location *LocationConfig `json:"location,omitempty"`+ AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`}// SearchOptions configures a search request.@@ -188,7 +188,12 @@ type AgentOptions struct {StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`Model *string `json:"model,omitempty"`Webhook *WebhookConfig `json:"webhook,omitempty"`- AuditMetadata map[string]string `json:"auditMetadata,omitempty"`+ AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`+}++// AuditMetadata identifies the user associated with a SIEM logging event.+type AuditMetadata struct {+ Username string `json:"username"`}// LocationConfig specifies geolocation for requests.diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.goindex 4afbb89a7..f7f230658 100644--- a/apps/go-sdk/options_test.go+++ b/apps/go-sdk/options_test.go@@ -89,7 +89,7 @@ func TestSearchOptionsSerializesHighlights(t *testing.T) {}func TestAuditMetadataSerializesAcrossRequestOptions(t *testing.T) {- metadata := map[string]string{"requestId": "req-123"}+ metadata := &AuditMetadata{Username: "[email protected]"}for name, options := range map[string]interface{}{"scrape": ScrapeOptions{AuditMetadata: metadata},"map": MapOptions{AuditMetadata: metadata},@@ -101,7 +101,7 @@ func TestAuditMetadataSerializesAcrossRequestOptions(t *testing.T) {if err != nil {t.Fatalf("Marshal options: %v", err)}- if !strings.Contains(string(payload), `"auditMetadata":{"requestId":"req-123"}`) {+ if !strings.Contains(string(payload), `"auditMetadata":{"username":"[email protected]"}`) {t.Fatalf("serialized options = %s", payload)}})diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.goindex 6aacae6d3..29a379e7e 100644--- a/apps/go-sdk/parse.go+++ b/apps/go-sdk/parse.go@@ -71,7 +71,7 @@ type ParseOptions struct {Proxy *string `json:"proxy,omitempty"`Integration *string `json:"integration,omitempty"`RedactPII *bool `json:"redactPII,omitempty"`- AuditMetadata map[string]string `json:"auditMetadata,omitempty"`+ AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`}diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 31f90dbfe..64a7295db 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.9.0"+const Version = "1.9.1"diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex b30daeb42..19f377982 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.12.0"+version = "1.12.1"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex c605b3da4..e80852fc5 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";- private static final String SDK_ORIGIN = "[email protected]";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.javaindex c2873512b..36c9b1ba4 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java@@ -2,8 +2,6 @@ package com.firecrawl.models;import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonProperty;-import java.util.Collections;-import java.util.HashMap;import java.util.List;import java.util.Map;@@ -21,7 +19,7 @@ public class AgentOptions {private Boolean strictConstrainToURLs;private String model;private WebhookConfig webhook;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private AgentOptions() {}@@ -34,7 +32,7 @@ public class AgentOptions {public String getModel() { return model; }public WebhookConfig getWebhook() { return webhook; }@JsonProperty("auditMetadata")- public Map<String, String> getAuditMetadata() { return auditMetadata; }+ public AuditMetadata getAuditMetadata() { return auditMetadata; }public static Builder builder() { return new Builder(); }@@ -47,7 +45,7 @@ public class AgentOptions {private Boolean strictConstrainToURLs;private String model;private WebhookConfig webhook;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private Builder() {}@@ -67,8 +65,8 @@ public class AgentOptions {public Builder model(String model) { this.model = model; return this; }/** Webhook configuration. */public Builder webhook(WebhookConfig webhook) { this.webhook = webhook; return this; }- /** Metadata to include with SIEM logging events. */- public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }+ /** User attribution to include with SIEM logging events. */+ public Builder auditMetadata(AuditMetadata auditMetadata) { this.auditMetadata = auditMetadata; return this; }public AgentOptions build() {if (prompt == null || prompt.isEmpty()) {@@ -83,9 +81,7 @@ public class AgentOptions {o.strictConstrainToURLs = this.strictConstrainToURLs;o.model = this.model;o.webhook = this.webhook;- o.auditMetadata = this.auditMetadata != null- ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata))- : null;+ o.auditMetadata = this.auditMetadata;return o;}}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/AuditMetadata.java b/apps/java-sdk/src/main/java/com/firecrawl/models/AuditMetadata.javanew file mode 100644index 000000000..0ab28a7c6--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/AuditMetadata.java@@ -0,0 +1,20 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonProperty;+import java.util.Objects;++/**+ * User attribution included with SIEM logging events.+ */+public final class AuditMetadata {+ private final String username;++ public AuditMetadata(String username) {+ this.username = Objects.requireNonNull(username, "username");+ }++ @JsonProperty("username")+ public String getUsername() {+ return username;+ }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.javaindex b4c14ce47..8b7c8ebcd 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java@@ -2,9 +2,6 @@ package com.firecrawl.models;import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonProperty;-import java.util.Collections;-import java.util.HashMap;-import java.util.Map;/*** Options for mapping (discovering URLs on) a website.@@ -20,7 +17,7 @@ public class MapOptions {private Integer timeout;private String integration;private LocationConfig location;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private MapOptions() {}@@ -34,7 +31,7 @@ public class MapOptions {public String getIntegration() { return integration; }public LocationConfig getLocation() { return location; }@JsonProperty("auditMetadata")- public Map<String, String> getAuditMetadata() { return auditMetadata; }+ public AuditMetadata getAuditMetadata() { return auditMetadata; }public static Builder builder() { return new Builder(); }@@ -47,7 +44,7 @@ public class MapOptions {private Integer timeout;private String integration;private LocationConfig location;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private Builder() {}@@ -67,8 +64,8 @@ public class MapOptions {public Builder integration(String integration) { this.integration = integration; return this; }/** Geolocation configuration. */public Builder location(LocationConfig location) { this.location = location; return this; }- /** Metadata to include with SIEM logging events. */- public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }+ /** User attribution to include with SIEM logging events. */+ public Builder auditMetadata(AuditMetadata auditMetadata) { this.auditMetadata = auditMetadata; return this; }public MapOptions build() {MapOptions o = new MapOptions();@@ -80,9 +77,7 @@ public class MapOptions {o.timeout = this.timeout;o.integration = this.integration;o.location = this.location;- o.auditMetadata = this.auditMetadata != null- ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata))- : null;+ o.auditMetadata = this.auditMetadata;return o;}}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.javaindex c8094b264..01b133df1 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java@@ -31,7 +31,7 @@ public class ParseOptions {private String integration;@JsonProperty("redactPII")private Boolean redactPII;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private ParseOptions() {}@@ -50,7 +50,7 @@ public class ParseOptions {@JsonProperty("redactPII")public Boolean getRedactPII() { return redactPII; }@JsonProperty("auditMetadata")- public Map<String, String> getAuditMetadata() { return auditMetadata; }+ public AuditMetadata getAuditMetadata() { return auditMetadata; }public static Builder builder() { return new Builder(); }@@ -69,7 +69,7 @@ public class ParseOptions {b.proxy = this.proxy;b.integration = this.integration;b.redactPII = this.redactPII;- b.auditMetadata = this.auditMetadata != null ? new HashMap<>(this.auditMetadata) : null;+ b.auditMetadata = this.auditMetadata;return b;}@@ -117,7 +117,7 @@ public class ParseOptions {private String proxy;private String integration;private Boolean redactPII;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private Builder() {}@@ -134,7 +134,7 @@ public class ParseOptions {public Builder proxy(String proxy) { this.proxy = proxy; return this; }public Builder integration(String integration) { this.integration = integration; return this; }public Builder redactPII(Boolean redactPII) { this.redactPII = redactPII; return this; }- public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }+ public Builder auditMetadata(AuditMetadata auditMetadata) { this.auditMetadata = auditMetadata; return this; }public ParseOptions build() {if (timeout != null && timeout <= 0) {@@ -168,7 +168,7 @@ public class ParseOptions {o.proxy = this.proxy;o.integration = this.integration;o.redactPII = this.redactPII;- o.auditMetadata = this.auditMetadata != null ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata)) : null;+ o.auditMetadata = this.auditMetadata;return o;}}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.javaindex e36050148..756001373 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java@@ -35,7 +35,7 @@ public class ScrapeOptions {private Boolean lockdown;@JsonProperty("redactPII")private Boolean redactPII;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private String integration;private ScrapeOptions() {}@@ -61,7 +61,7 @@ public class ScrapeOptions {@JsonProperty("redactPII")public Boolean getRedactPII() { return redactPII; }@JsonProperty("auditMetadata")- public Map<String, String> getAuditMetadata() { return auditMetadata; }+ public AuditMetadata getAuditMetadata() { return auditMetadata; }public String getIntegration() { return integration; }public static Builder builder() { return new Builder(); }@@ -87,7 +87,7 @@ public class ScrapeOptions {b.storeInCache = this.storeInCache;b.lockdown = this.lockdown;b.redactPII = this.redactPII;- b.auditMetadata = this.auditMetadata != null ? new HashMap<>(this.auditMetadata) : null;+ b.auditMetadata = this.auditMetadata;b.integration = this.integration;return b;}@@ -112,7 +112,7 @@ public class ScrapeOptions {private Boolean storeInCache;private Boolean lockdown;private Boolean redactPII;- private Map<String, String> auditMetadata;+ private AuditMetadata auditMetadata;private String integration;private Builder() {}@@ -178,8 +178,8 @@ public class ScrapeOptions {/** Redact personally identifiable information from returned content. */public Builder redactPII(Boolean redactPII) { this.redactPII = redactPII; return this; }- /** Metadata to include with SIEM logging events. */- public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }+ /** User attribution to include with SIEM logging events. */+ public Builder auditMetadata(AuditMetadata auditMetadata) { this.auditMetadata = auditMetadata; return this; }/** Integration identifier. */public Builder integration(String integration) { this.integration = integration; return this; }@@ -205,7 +205,7 @@ public class ScrapeOptions {o.storeInCache = this.storeInCache;o.lockdown = this.lockdown;o.redactPII = this.redactPII;- o.auditMetadata = this.auditMetadata != null ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata)) : null;+ o.auditMetadata = this.auditMetadata;o.integration = this.integration;return o;}diff --git a/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java b/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.javaindex 354ed6b61..4ab9b2c06 100644--- a/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java+++ b/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java@@ -2,16 +2,15 @@ package com.firecrawl;import com.fasterxml.jackson.databind.ObjectMapper;import com.firecrawl.models.AgentOptions;+import com.firecrawl.models.AuditMetadata;import com.firecrawl.models.MapOptions;import com.firecrawl.models.ParseOptions;import com.firecrawl.models.ScrapeOptions;import org.junit.jupiter.api.Test;-import java.util.HashMap;import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;-import static org.junit.jupiter.api.Assertions.assertThrows;class AuditMetadataTest {@@ -19,7 +18,7 @@ class AuditMetadataTest {@Testvoid serializesAuditMetadataAcrossRequestOptions() {- Map<String, String> metadata = Map.of("requestId", "req-123");+ AuditMetadata metadata = new AuditMetadata("[email protected]");assertAuditMetadata(ScrapeOptions.builder().auditMetadata(metadata).build());assertAuditMetadata(MapOptions.builder().auditMetadata(metadata).build());@@ -27,28 +26,8 @@ class AuditMetadataTest {assertAuditMetadata(ParseOptions.builder().auditMetadata(metadata).build());}- @Test- void mapAndAgentOptionsDefensivelyCopyAuditMetadata() {- Map<String, String> metadata = new HashMap<>();- metadata.put("requestId", "req-123");-- MapOptions map = MapOptions.builder().auditMetadata(metadata).build();- AgentOptions agent = AgentOptions.builder()- .prompt("find pricing")- .auditMetadata(metadata)- .build();- metadata.put("requestId", "mutated");-- assertEquals("req-123", map.getAuditMetadata().get("requestId"));- assertEquals("req-123", agent.getAuditMetadata().get("requestId"));- assertThrows(UnsupportedOperationException.class,- () -> map.getAuditMetadata().put("requestId", "mutated"));- assertThrows(UnsupportedOperationException.class,- () -> agent.getAuditMetadata().put("requestId", "mutated"));- }-private void assertAuditMetadata(Object options) {Map<?, ?> body = mapper.convertValue(options, Map.class);- assertEquals(Map.of("requestId", "req-123"), body.get("auditMetadata"));+ assertEquals(Map.of("username", "[email protected]"), body.get("auditMetadata"));}}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex ecfd6eba8..653253e4c 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.31.0",+ "version": "4.31.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.tsindex 620fde00a..2ae9adebe 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts@@ -7,7 +7,7 @@ import { map } from "../../../v2/methods/map";import { startExtract } from "../../../v2/methods/extract";import { startAgent } from "../../../v2/methods/agent";-const auditMetadata = { requestId: "req-123" };+const auditMetadata = { username: "[email protected]" };function makeHttp(data: Record<string, unknown>) {const post = jest.fn(async () => ({ status: 200, data }));diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 665130d9b..926919657 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -114,6 +114,10 @@ export interface LocationConfig {languages?: string[];}+export interface AuditMetadata {+ username: string;+}+/*** Parameters for scraping operations.* Defines the options and configurations available for scraping web content.@@ -135,7 +139,7 @@ export interface CrawlScrapeOptions {storeInCache?: boolean;maxAge?: number;parsePDF?: boolean;- auditMetadata?: Record<string, string>;+ auditMetadata?: AuditMetadata;}export type Action = {@@ -311,7 +315,7 @@ export interface MapParams {timeout?: number;useIndex?: boolean;location?: LocationConfig;- auditMetadata?: Record<string, string>;+ auditMetadata?: AuditMetadata;}/**diff --git a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts b/apps/js-sdk/firecrawl/src/v2/methods/agent.tsindex 16cdcb388..d87113542 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts@@ -1,4 +1,4 @@-import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type ThreatProtectionOptions } from "../types";+import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";import { HttpClient } from "../utils/httpClient";import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";@@ -15,7 +15,7 @@ function prepareAgentPayload(args: {model?: "spark-1-pro" | "spark-1-mini";webhook?: string | AgentWebhookConfig;threatProtection?: ThreatProtectionOptions;- auditMetadata?: Record<string, string>;+ auditMetadata?: AuditMetadata;}): Record<string, unknown> {const body: Record<string, unknown> = {};if (args.urls) body.urls = args.urls;diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 24583af32..c7ebc8eed 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -183,6 +183,10 @@ export type ActionOption =| ExecuteJavascriptAction| PDFAction;+export interface AuditMetadata {+ username: string;+}+export interface ScrapeOptions {formats?: FormatOption[];headers?: Record<string, string>;@@ -209,7 +213,7 @@ export interface ScrapeOptions {lockdown?: boolean;redactPII?: boolean | RedactPIIOptions;threatProtection?: ThreatProtectionOptions;- auditMetadata?: Record<string, string>;+ auditMetadata?: AuditMetadata;profile?: {name: string;saveChanges?: boolean;@@ -797,7 +801,7 @@ export interface MapOptions {origin?: string;location?: LocationConfig;threatProtection?: ThreatProtectionOptions;- auditMetadata?: Record<string, string>;+ auditMetadata?: AuditMetadata;}export type FeedbackRating = "good" | "partial" | "bad";diff --git a/apps/php-sdk/src/Models/AgentOptions.php b/apps/php-sdk/src/Models/AgentOptions.phpindex 381268549..439ea2c8c 100644--- a/apps/php-sdk/src/Models/AgentOptions.php+++ b/apps/php-sdk/src/Models/AgentOptions.php@@ -9,7 +9,7 @@ final class AgentOptions/*** @param list<string>|null $urls* @param array<string, mixed>|null $schema- * @param array<string, string>|null $auditMetadata+ * @param AuditMetadata|null $auditMetadata*/private function __construct(private readonly ?array $urls = null,@@ -20,14 +20,13 @@ final class AgentOptionsprivate readonly ?bool $strictConstrainToURLs = null,private readonly ?string $model = null,private readonly ?WebhookConfig $webhook = null,- /** @var array<string, string>|null */- private readonly ?array $auditMetadata = null,+ private readonly ?AuditMetadata $auditMetadata = null,) {}/*** @param list<string>|null $urls* @param array<string, mixed>|null $schema- * @param array<string, string>|null $auditMetadata+ * @param AuditMetadata|null $auditMetadata*/public static function with(?array $urls = null,@@ -38,7 +37,7 @@ final class AgentOptions?bool $strictConstrainToURLs = null,?string $model = null,?WebhookConfig $webhook = null,- ?array $auditMetadata = null,+ ?AuditMetadata $auditMetadata = null,): self {return new self($urls, $prompt, $schema, $integration,@@ -58,7 +57,7 @@ final class AgentOptions'strictConstrainToURLs' => $this->strictConstrainToURLs,'model' => $this->model,'webhook' => $this->webhook?->toArray(),- 'auditMetadata' => $this->auditMetadata,+ 'auditMetadata' => $this->auditMetadata?->toArray(),];return array_filter($fields, fn (mixed $v): bool => $v !== null);diff --git a/apps/php-sdk/src/Models/AuditMetadata.php b/apps/php-sdk/src/Models/AuditMetadata.phpnew file mode 100644index 000000000..2a951d384--- /dev/null+++ b/apps/php-sdk/src/Models/AuditMetadata.php@@ -0,0 +1,28 @@+<?php++declare(strict_types=1);++namespace Firecrawl\Models;++final class AuditMetadata+{+ private function __construct(+ private readonly string $username,+ ) {}++ public static function with(string $username): self+ {+ return new self($username);+ }++ /** @return array{username: string} */+ public function toArray(): array+ {+ return ['username' => $this->username];+ }++ public function getUsername(): string+ {+ return $this->username;+ }+}diff --git a/apps/php-sdk/src/Models/MapOptions.php b/apps/php-sdk/src/Models/MapOptions.phpindex 397847898..fc2810ebe 100644--- a/apps/php-sdk/src/Models/MapOptions.php+++ b/apps/php-sdk/src/Models/MapOptions.php@@ -6,7 +6,6 @@ namespace Firecrawl\Models;final class MapOptions{- /** @param array<string, string>|null $auditMetadata */private function __construct(private readonly ?string $search = null,private readonly ?string $sitemap = null,@@ -16,11 +15,9 @@ final class MapOptionsprivate readonly ?int $timeout = null,private readonly ?string $integration = null,private readonly ?LocationConfig $location = null,- /** @var array<string, string>|null */- private readonly ?array $auditMetadata = null,+ private readonly ?AuditMetadata $auditMetadata = null,) {}- /** @param array<string, string>|null $auditMetadata */public static function with(?string $search = null,?string $sitemap = null,@@ -30,7 +27,7 @@ final class MapOptions?int $timeout = null,?string $integration = null,?LocationConfig $location = null,- ?array $auditMetadata = null,+ ?AuditMetadata $auditMetadata = null,): self {return new self($search, $sitemap, $includeSubdomains, $ignoreQueryParameters,@@ -50,7 +47,7 @@ final class MapOptions'timeout' => $this->timeout,'integration' => $this->integration,'location' => $this->location?->toArray(),- 'auditMetadata' => $this->auditMetadata,+ 'auditMetadata' => $this->auditMetadata?->toArray(),];return array_filter($fields, fn (mixed $v): bool => $v !== null);diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.phpindex c8a3f0fc8..06c14db89 100644--- a/apps/php-sdk/src/Models/ParseOptions.php+++ b/apps/php-sdk/src/Models/ParseOptions.php@@ -32,7 +32,7 @@ final class ParseOptions* @param list<string>|null $includeTags* @param list<string>|null $excludeTags* @param list<mixed>|null $parsers- * @param array<string, string>|null $auditMetadata+ * @param AuditMetadata|null $auditMetadata*/private function __construct(private readonly ?array $formats = null,@@ -48,8 +48,7 @@ final class ParseOptionsprivate readonly ?string $proxy = null,private readonly ?string $integration = null,private readonly ?bool $redactPII = null,- /** @var array<string, string>|null */- private readonly ?array $auditMetadata = null,+ private readonly ?AuditMetadata $auditMetadata = null,) {}/**@@ -58,7 +57,7 @@ final class ParseOptions* @param list<string>|null $includeTags* @param list<string>|null $excludeTags* @param list<mixed>|null $parsers- * @param array<string, string>|null $auditMetadata+ * @param AuditMetadata|null $auditMetadata*/public static function with(?array $formats = null,@@ -74,7 +73,7 @@ final class ParseOptions?string $proxy = null,?string $integration = null,?bool $redactPII = null,- ?array $auditMetadata = null,+ ?AuditMetadata $auditMetadata = null,): self {if ($timeout !== null && $timeout <= 0) {throw new FirecrawlException('timeout must be positive');@@ -142,7 +141,7 @@ final class ParseOptions'proxy' => $this->proxy,'integration' => $this->integration,'redactPII' => $this->redactPII,- 'auditMetadata' => $this->auditMetadata,+ 'auditMetadata' => $this->auditMetadata?->toArray(),];foreach ($fields as $key => $value) {@@ -242,8 +241,7 @@ final class ParseOptionsreturn $this->integration;}- /** @return array<string, string>|null */- public function getAuditMetadata(): ?array+ public function getAuditMetadata(): ?AuditMetadata{return $this->auditMetadata;}diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.phpindex d264cd162..5cc3ad42b 100644--- a/apps/php-sdk/src/Models/ScrapeOptions.php+++ b/apps/php-sdk/src/Models/ScrapeOptions.php@@ -13,7 +13,7 @@ final class ScrapeOptions* @param list<string>|null $excludeTags* @param list<mixed>|null $parsers* @param list<array<string, mixed>>|null $actions- * @param array<string, string>|null $auditMetadata+ * @param AuditMetadata|null $auditMetadata*/private function __construct(private readonly ?array $formats = null,@@ -40,8 +40,7 @@ final class ScrapeOptionsprivate readonly ?array $profile = null,private readonly ?bool $changeTracking = null,private readonly ?bool $redactPII = null,- /** @var array<string, string>|null */- private readonly ?array $auditMetadata = null,+ private readonly ?AuditMetadata $auditMetadata = null,) {}/**@@ -52,7 +51,7 @@ final class ScrapeOptions* @param list<mixed>|null $parsers* @param list<array<string, mixed>>|null $actions* @param array<string, string>|null $profile- * @param array<string, string>|null $auditMetadata+ * @param AuditMetadata|null $auditMetadata*/public static function with(?array $formats = null,@@ -78,7 +77,7 @@ final class ScrapeOptions?array $profile = null,?bool $changeTracking = null,?bool $redactPII = null,- ?array $auditMetadata = null,+ ?AuditMetadata $auditMetadata = null,): self {return new self($formats, $headers, $includeTags, $excludeTags, $onlyMainContent,@@ -131,7 +130,7 @@ final class ScrapeOptions'profile' => $this->profile,'changeTracking' => $this->changeTracking,'redactPII' => $this->redactPII,- 'auditMetadata' => $this->auditMetadata,+ 'auditMetadata' => $this->auditMetadata?->toArray(),];foreach ($fields as $key => $value) {@@ -265,8 +264,7 @@ final class ScrapeOptionsreturn $this->changeTracking;}- /** @return array<string, string>|null */- public function getAuditMetadata(): ?array+ public function getAuditMetadata(): ?AuditMetadata{return $this->auditMetadata;}diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex d8f95f0bc..40f902912 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.10.0';+ public const SDK_VERSION = '1.10.1';}diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.phpindex c78dbc863..9d0d5aba7 100644--- a/apps/php-sdk/tests/Unit/ModelsTest.php+++ b/apps/php-sdk/tests/Unit/ModelsTest.php@@ -11,6 +11,7 @@ use Firecrawl\Models\BatchScrapeJob;use Firecrawl\Models\CrawlJob;use Firecrawl\Models\HighlightsFormat;use Firecrawl\Models\AgentOptions;+use Firecrawl\Models\AuditMetadata;use Firecrawl\Models\MapOptions;use Firecrawl\Models\ParseOptions;use Firecrawl\Models\QueryFormat;@@ -344,17 +345,18 @@ it('serializes redactPII in ScrapeOptions', function (): void {});it('serializes audit metadata across request options', function (): void {- $metadata = ['requestId' => 'req-123'];+ $metadata = AuditMetadata::with('[email protected]');+ $serialized = ['username' => '[email protected]'];$scrape = ScrapeOptions::with(auditMetadata: $metadata);- expect($scrape->toArray())->toMatchArray(['auditMetadata' => $metadata]);+ expect($scrape->toArray())->toMatchArray(['auditMetadata' => $serialized]);expect($scrape->getAuditMetadata())->toBe($metadata);expect(MapOptions::with(auditMetadata: $metadata)->toArray())- ->toMatchArray(['auditMetadata' => $metadata]);+ ->toMatchArray(['auditMetadata' => $serialized]);expect(AgentOptions::with(prompt: 'find pricing', auditMetadata: $metadata)->toArray())- ->toMatchArray(['auditMetadata' => $metadata]);+ ->toMatchArray(['auditMetadata' => $serialized]);$parse = ParseOptions::with(auditMetadata: $metadata);- expect($parse->toArray())->toMatchArray(['auditMetadata' => $metadata]);+ expect($parse->toArray())->toMatchArray(['auditMetadata' => $serialized]);expect($parse->getAuditMetadata())->toBe($metadata);});diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 5ac05814e..7a1a6e623 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -9,6 +9,7 @@ import osfrom .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlAppfrom .v2.watcher import Watcherfrom .v2.watcher_async import AsyncWatcher+from .v2.types import AuditMetadatafrom .v1 import (V1FirecrawlApp,AsyncV1FirecrawlApp,@@ -17,7 +18,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.33.0"+__version__ = "4.33.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")@@ -79,6 +80,7 @@ __all__ = ['AsyncFirecrawlApp','Watcher','AsyncWatcher',+ 'AuditMetadata','V1FirecrawlApp','AsyncV1FirecrawlApp','V1JsonConfig',diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.pyindex 685591abe..ff84ff751 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py@@ -1,34 +1,38 @@import json+import pytest+from pydantic import ValidationError+from firecrawl.v2.methods.agent import _prepare_agent_requestfrom firecrawl.v2.methods.aio.batch import _prepare as prepare_async_batch_requestfrom firecrawl.v2.methods.batch import prepare_batch_scrape_requestfrom firecrawl.v2.methods.map import _prepare_map_requestfrom firecrawl.v2.methods.parse import _prepare_parse_request-from firecrawl.v2.types import MapOptions, ParseOptions, ScrapeOptions+from firecrawl.v2.types import AuditMetadata, MapOptions, ParseOptions, ScrapeOptionsfrom firecrawl.v2.utils.validation import prepare_scrape_options-AUDIT_METADATA = {"requestId": "req-123", "actor": "sdk-test"}+AUDIT_METADATA = AuditMetadata(username="[email protected]")+SERIALIZED_AUDIT_METADATA = {"username": "[email protected]"}def test_scrape_and_batch_audit_metadata_request_preparation():options = ScrapeOptions(audit_metadata=AUDIT_METADATA)- assert prepare_scrape_options(options)["auditMetadata"] == AUDIT_METADATA+ assert prepare_scrape_options(options)["auditMetadata"] == SERIALIZED_AUDIT_METADATAassert (prepare_batch_scrape_request(["https://example.com"],audit_metadata=AUDIT_METADATA,)["auditMetadata"]- == AUDIT_METADATA+ == SERIALIZED_AUDIT_METADATA)assert (prepare_async_batch_request(["https://example.com"],audit_metadata=AUDIT_METADATA,)["auditMetadata"]- == AUDIT_METADATA+ == SERIALIZED_AUDIT_METADATA)@@ -38,7 +42,7 @@ def test_map_and_agent_audit_metadata_request_preparation():"https://example.com",MapOptions(audit_metadata=AUDIT_METADATA),)["auditMetadata"]- == AUDIT_METADATA+ == SERIALIZED_AUDIT_METADATA)assert (_prepare_agent_request(@@ -46,7 +50,7 @@ def test_map_and_agent_audit_metadata_request_preparation():prompt="Extract the page",audit_metadata=AUDIT_METADATA,)["auditMetadata"]- == AUDIT_METADATA+ == SERIALIZED_AUDIT_METADATA)@@ -58,4 +62,9 @@ def test_parse_audit_metadata_request_preparation():content_type="text/html",)- assert json.loads(fields["options"])["auditMetadata"] == AUDIT_METADATA+ assert json.loads(fields["options"])["auditMetadata"] == SERIALIZED_AUDIT_METADATA+++def test_audit_metadata_rejects_unsupported_fields():+ with pytest.raises(ValidationError):+ AuditMetadata(username="[email protected]", session="session-123")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex b0a21caba..64371d85b 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -42,6 +42,7 @@ from .types import (Location,PaginationConfig,AgentOptions,+ AuditMetadata,ThreatProtectionOptions,Monitor,MonitorCheck,@@ -161,7 +162,7 @@ class FirecrawlClient:lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,integration: Optional[str] = None,) -> Document:"""@@ -486,7 +487,7 @@ class FirecrawlClient:lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,regex_on_full_url: bool = False,deduplicate_similar_urls: bool = True,zero_data_retention: bool = False,@@ -648,7 +649,7 @@ class FirecrawlClient:lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,regex_on_full_url: bool = False,deduplicate_similar_urls: bool = True,zero_data_retention: bool = False,@@ -852,7 +853,7 @@ class FirecrawlClient:integration: Optional[str] = None,location: Optional[Location] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> MapData:"""Map a URL and return discovered links.@@ -1168,7 +1169,7 @@ class FirecrawlClient:store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,webhook: Optional[Union[str, WebhookConfig]] = None,append_to_id: Optional[str] = None,ignore_invalid_urls: Optional[bool] = None,@@ -1345,7 +1346,7 @@ class FirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,):"""Start an agent job (non-blocking).@@ -1391,7 +1392,7 @@ class FirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,):"""Run an agent and wait until completion.@@ -1604,7 +1605,7 @@ class FirecrawlClient:store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,webhook: Optional[Union[str, WebhookConfig]] = None,append_to_id: Optional[str] = None,ignore_invalid_urls: Optional[bool] = None,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 9ea7eaa4a..4b45e6207 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -38,6 +38,7 @@ from .types import (Location,PaginationConfig,ThreatProtectionOptions,+ AuditMetadata,Monitor,MonitorCheck,MonitorCheckDetail,@@ -399,7 +400,7 @@ class AsyncFirecrawlClient:timeout: Optional[int] = None,integration: Optional[str] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> MapData:options = MapOptions(search=search,@@ -682,7 +683,7 @@ class AsyncFirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,):return await async_agent.agent(self.async_http_client,@@ -715,7 +716,7 @@ class AsyncFirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,):return await async_agent.start_agent(self.async_http_client,diff --git a/apps/python-sdk/firecrawl/v2/methods/agent.py b/apps/python-sdk/firecrawl/v2/methods/agent.pyindex 6823ce3f6..64a22dc98 100644--- a/apps/python-sdk/firecrawl/v2/methods/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/agent.py@@ -1,7 +1,7 @@from typing import Any, Dict, List, Literal, Optional, Unionimport time-from ..types import AgentResponse, AgentWebhookConfig, ThreatProtectionOptions+from ..types import AgentResponse, AgentWebhookConfig, AuditMetadata, ThreatProtectionOptionsfrom ..utils.http_client import HttpClientfrom ..utils.error_handler import handle_response_errorfrom ..utils.validation import _normalize_schema@@ -18,7 +18,7 @@ def _prepare_agent_request(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -51,7 +51,7 @@ def _prepare_agent_request(by_alias=True, exclude_none=True)if audit_metadata is not None:- body["auditMetadata"] = audit_metadata+ body["auditMetadata"] = audit_metadata.model_dump()return body@@ -76,7 +76,7 @@ def start_agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -136,7 +136,7 @@ def agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> AgentResponse:started = start_agent(client,diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.pyindex 86eecc076..0b7fb67b1 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py@@ -1,7 +1,7 @@from typing import Any, Dict, List, Literal, Optional, Unionimport asyncio-from ...types import AgentResponse, AgentWebhookConfig, ThreatProtectionOptions+from ...types import AgentResponse, AgentWebhookConfig, AuditMetadata, ThreatProtectionOptionsfrom ...utils.error_handler import handle_response_errorfrom ...utils.http_client_async import AsyncHttpClientfrom ...utils.validation import _normalize_schema@@ -18,7 +18,7 @@ def _prepare_agent_request(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -51,7 +51,7 @@ def _prepare_agent_request(by_alias=True, exclude_none=True)if audit_metadata is not None:- body["auditMetadata"] = audit_metadata+ body["auditMetadata"] = audit_metadata.model_dump()return body@@ -76,7 +76,7 @@ async def start_agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -136,7 +136,7 @@ async def agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> AgentResponse:started = await start_agent(client,diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.pyindex 1cdd1a13a..8d361dad6 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py@@ -41,7 +41,7 @@ def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwarif opts:payload.update(opts)if (v := kwargs.get("audit_metadata")) is not None:- payload["auditMetadata"] = v+ payload["auditMetadata"] = v.model_dump()if (w := kwargs.get("webhook")) is not None:payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)if (v := kwargs.get("append_to_id")) is not None:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/map.py b/apps/python-sdk/firecrawl/v2/methods/aio/map.pyindex 4ef0e35ae..09eff4be8 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/map.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/map.py@@ -33,7 +33,7 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dictby_alias=True, exclude_none=True)if options.audit_metadata is not None:- data["auditMetadata"] = options.audit_metadata+ data["auditMetadata"] = options.audit_metadata.model_dump()payload.update(data)return payloaddiff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex f81768018..20a43c1b8 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -12,6 +12,7 @@ from ..types import (Document,WebhookConfig,PaginationConfig,+ AuditMetadata,)from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_optionsfrom ..utils.normalize import normalize_document_input@@ -53,7 +54,7 @@ def start_batch_scrape(max_concurrency: Optional[int] = None,zero_data_retention: Optional[bool] = None,integration: Optional[str] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,idempotency_key: Optional[str] = None,) -> BatchScrapeResponse:"""@@ -343,7 +344,7 @@ def batch_scrape(max_concurrency: Optional[int] = None,zero_data_retention: Optional[bool] = None,integration: Optional[str] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,idempotency_key: Optional[str] = None,poll_interval: int = 2,timeout: Optional[int] = None@@ -428,7 +429,7 @@ def prepare_batch_scrape_request(max_concurrency: Optional[int] = None,zero_data_retention: Optional[bool] = None,integration: Optional[str] = None,- audit_metadata: Optional[Dict[str, str]] = None,+ audit_metadata: Optional[AuditMetadata] = None,) -> dict:"""Prepare a batch scrape request payload.@@ -449,7 +450,7 @@ def prepare_batch_scrape_request(if scrape_data:request_data.update(scrape_data)if audit_metadata is not None:- request_data["auditMetadata"] = audit_metadata+ request_data["auditMetadata"] = audit_metadata.model_dump()# Batch-specific fieldsif webhook is not None:diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.pyindex 7da8073ba..7ebff6414 100644--- a/apps/python-sdk/firecrawl/v2/methods/map.py+++ b/apps/python-sdk/firecrawl/v2/methods/map.py@@ -40,7 +40,7 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dictby_alias=True, exclude_none=True)if options.audit_metadata is not None:- data["auditMetadata"] = options.audit_metadata+ data["auditMetadata"] = options.audit_metadata.model_dump()payload.update(data)return payloaddiff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex f0012742f..08e9d9ba0 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -778,6 +778,14 @@ class ThreatProtectionOptions(BaseModel):model_config = {"populate_by_name": True}+class AuditMetadata(BaseModel):+ """User attribution included with SIEM logging events."""++ username: str = Field(max_length=1024)++ model_config = {"extra": "forbid"}++class ScrapeOptions(BaseModel):"""Options for scraping operations."""@@ -822,7 +830,7 @@ class ScrapeOptions(BaseModel):threat_protection: Optional[ThreatProtectionOptions] = Field(default=None, alias="threatProtection")- audit_metadata: Optional[Dict[str, str]] = Field(+ audit_metadata: Optional[AuditMetadata] = Field(default=None, alias="auditMetadata")profile: Optional[Dict[str, Any]] = None@@ -1057,7 +1065,7 @@ class MapOptions(BaseModel):integration: Optional[str] = Nonelocation: Optional["Location"] = Nonethreat_protection: Optional[ThreatProtectionOptions] = None- audit_metadata: Optional[Dict[str, str]] = None+ audit_metadata: Optional[AuditMetadata] = Noneclass MapRequest(BaseModel):diff --git a/apps/ruby-sdk/lib/firecrawl.rb b/apps/ruby-sdk/lib/firecrawl.rbindex 6514a2182..3ba8cd0dd 100644--- a/apps/ruby-sdk/lib/firecrawl.rb+++ b/apps/ruby-sdk/lib/firecrawl.rb@@ -7,6 +7,7 @@ require_relative "firecrawl/models/query_format"require_relative "firecrawl/models/product_profile"require_relative "firecrawl/models/menu_profile"require_relative "firecrawl/models/document"+require_relative "firecrawl/models/audit_metadata"require_relative "firecrawl/models/scrape_options"require_relative "firecrawl/models/crawl_options"require_relative "firecrawl/models/crawl_response"diff --git a/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb b/apps/ruby-sdk/lib/firecrawl/models/agent_options.rbindex c6f3313be..c87edef31 100644--- a/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb@@ -13,6 +13,9 @@ module Firecrawldef initialize(**kwargs)FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }+ if audit_metadata && !audit_metadata.is_a?(AuditMetadata)+ raise ArgumentError, "audit_metadata must be an AuditMetadata"+ endraise ArgumentError, "Agent prompt is required" if prompt.nil? || prompt.empty?end@@ -26,7 +29,7 @@ module Firecrawl"strictConstrainToURLs" => strict_constrain_to_urls,"model" => model,"webhook" => webhook.is_a?(Hash) ? webhook : webhook&.to_h,- "auditMetadata" => audit_metadata,+ "auditMetadata" => audit_metadata&.to_h,}.compactendenddiff --git a/apps/ruby-sdk/lib/firecrawl/models/audit_metadata.rb b/apps/ruby-sdk/lib/firecrawl/models/audit_metadata.rbnew file mode 100644index 000000000..5c2938acd--- /dev/null+++ b/apps/ruby-sdk/lib/firecrawl/models/audit_metadata.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++module Firecrawl+ module Models+ # User attribution included with SIEM logging events.+ class AuditMetadata+ attr_reader :username++ def initialize(username:)+ raise ArgumentError, "username must be a string" unless username.is_a?(String)++ @username = username+ end++ def to_h+ { "username" => username }+ end+ end+ end+enddiff --git a/apps/ruby-sdk/lib/firecrawl/models/map_options.rb b/apps/ruby-sdk/lib/firecrawl/models/map_options.rbindex 1f3824f68..7ed4e0035 100644--- a/apps/ruby-sdk/lib/firecrawl/models/map_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/map_options.rb@@ -13,6 +13,9 @@ module Firecrawldef initialize(**kwargs)FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }+ if audit_metadata && !audit_metadata.is_a?(AuditMetadata)+ raise ArgumentError, "audit_metadata must be an AuditMetadata"+ endenddef to_h@@ -25,7 +28,7 @@ module Firecrawl"timeout" => timeout,"integration" => integration,"location" => location.is_a?(Hash) ? location : location&.to_h,- "auditMetadata" => audit_metadata,+ "auditMetadata" => audit_metadata&.to_h,}.compactendenddiff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rbindex 7ba9235af..cb7310e95 100644--- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb@@ -20,6 +20,9 @@ module Firecrawldef initialize(**kwargs)FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }+ if audit_metadata && !audit_metadata.is_a?(AuditMetadata)+ raise ArgumentError, "audit_metadata must be an AuditMetadata"+ endvalidate!end@@ -40,7 +43,7 @@ module Firecrawl"integration" => integration,"redactPII" => redact_pii,"jsonOptions" => json_options.is_a?(Hash) ? json_options : json_options&.to_h,- "auditMetadata" => audit_metadata,+ "auditMetadata" => audit_metadata&.to_h,}.compactenddiff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rbindex d860268b9..20c050555 100644--- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb@@ -15,6 +15,9 @@ module Firecrawldef initialize(**kwargs)FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }+ if audit_metadata && !audit_metadata.is_a?(AuditMetadata)+ raise ArgumentError, "audit_metadata must be an AuditMetadata"+ end@skip_tls_verification = false if @skip_tls_verification.nil?end@@ -40,7 +43,7 @@ module Firecrawl"lockdown" => lockdown,"redactPII" => redact_pii,"integration" => integration,- "auditMetadata" => audit_metadata,+ "auditMetadata" => audit_metadata&.to_h,}.compactenddiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex 7706c5496..c53228978 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.11.0"+ VERSION = "1.11.1"enddiff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex 0165206cc..598635490 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -770,20 +770,27 @@ class ClientTest < Minitest::Testenddef test_audit_metadata_to_h- metadata = { "requestId" => "req-123" }+ metadata = Firecrawl::Models::AuditMetadata.new(username: "[email protected]")+ serialized = { "username" => "[email protected]" }- assert_equal metadata, Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata).to_h["auditMetadata"]- assert_equal metadata, Firecrawl::Models::MapOptions.new(audit_metadata: metadata).to_h["auditMetadata"]- assert_equal metadata,+ assert_equal serialized, Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata).to_h["auditMetadata"]+ assert_equal serialized, Firecrawl::Models::MapOptions.new(audit_metadata: metadata).to_h["auditMetadata"]+ assert_equal serialized,Firecrawl::Models::AgentOptions.new(prompt: "find pricing",audit_metadata: metadata).to_h["auditMetadata"]- assert_equal metadata, Firecrawl::Models::ParseOptions.new(audit_metadata: metadata).to_h["auditMetadata"]+ assert_equal serialized, Firecrawl::Models::ParseOptions.new(audit_metadata: metadata).to_h["auditMetadata"]crawl = Firecrawl::Models::CrawlOptions.new(scrape_options: Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata))- assert_equal metadata, crawl.to_h.dig("scrapeOptions", "auditMetadata")+ assert_equal serialized, crawl.to_h.dig("scrapeOptions", "auditMetadata")+ end++ def test_audit_metadata_rejects_arbitrary_hashes+ assert_raises(ArgumentError) do+ Firecrawl::Models::ScrapeOptions.new(audit_metadata: { "session" => "session-123" })+ endenddef test_batch_scrape_options_to_hdiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 1918d7a5e..f37994035 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.12.0"+version = "2.12.1"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 3bdfa51a4..a44518286 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.12.0"+version = "2.12.1"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/agent.rs b/apps/rust-sdk/src/agent.rsindex a4c18b5a2..ae6cb077a 100644--- a/apps/rust-sdk/src/agent.rs+++ b/apps/rust-sdk/src/agent.rs@@ -3,13 +3,11 @@//! The Agent endpoint provides autonomous web browsing capabilities using AI//! to accomplish complex tasks that may require multiple page interactions.-use serde::{Deserialize, Serialize};-use serde_json::Value;-use std::collections::HashMap;-use crate::client::Client;use crate::types::{AgentModel, AgentWebhookConfig};-use crate::FirecrawlError;+use crate::{AuditMetadata, FirecrawlError};+use serde::{Deserialize, Serialize};+use serde_json::Value;/// Options for running an agent task.#[serde_with::skip_serializing_none]@@ -40,8 +38,8 @@ pub struct AgentOptions {/// Webhook configuration for agent notifications.pub webhook: Option<AgentWebhookConfig>,- /// Metadata to include with SIEM logging events.- pub audit_metadata: Option<HashMap<String, String>>,+ /// User attribution to include with SIEM logging events.+ pub audit_metadata: Option<AuditMetadata>,/// Poll interval for synchronous agent execution (milliseconds).#[serde(skip)]diff --git a/apps/rust-sdk/src/audit_metadata.rs b/apps/rust-sdk/src/audit_metadata.rsnew file mode 100644index 000000000..5b9395e71--- /dev/null+++ b/apps/rust-sdk/src/audit_metadata.rs@@ -0,0 +1,7 @@+use serde::{Deserialize, Serialize};++/// User attribution included with SIEM logging events.+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)]+pub struct AuditMetadata {+ pub username: String,+}diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rsindex d71a0bd97..b218bc3df 100644--- a/apps/rust-sdk/src/lib.rs+++ b/apps/rust-sdk/src/lib.rs@@ -21,6 +21,7 @@ pub mod error;pub(crate) mod serde_helpers;mod agent;+mod audit_metadata;mod batch_scrape;mod client;mod crawl;@@ -33,6 +34,7 @@ mod search;mod types;pub use agent::*;+pub use audit_metadata::*;pub use batch_scrape::*;pub use client::Client;pub use crawl::*;diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rsindex 1d686ae78..a49b10f8b 100644--- a/apps/rust-sdk/src/map.rs+++ b/apps/rust-sdk/src/map.rs@@ -1,11 +1,9 @@//! Map endpoint for Firecrawl API v2.-use serde::{Deserialize, Serialize};-use std::collections::HashMap;-use crate::client::Client;use crate::types::{LocationConfig, SearchResultWeb, SitemapMode};-use crate::FirecrawlError;+use crate::{AuditMetadata, FirecrawlError};+use serde::{Deserialize, Serialize};/// Options for mapping a URL.#[serde_with::skip_serializing_none]@@ -36,8 +34,8 @@ pub struct MapOptions {/// Location configuration for proxy routing.pub location: Option<LocationConfig>,- /// Metadata to include with SIEM logging events.- pub audit_metadata: Option<HashMap<String, String>>,+ /// User attribution to include with SIEM logging events.+ pub audit_metadata: Option<AuditMetadata>,}/// Request body for map endpoint.diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rsindex a6a4643c2..714cf1e23 100644--- a/apps/rust-sdk/src/parse.rs+++ b/apps/rust-sdk/src/parse.rs@@ -9,7 +9,7 @@ use std::collections::HashMap;use super::client::Client;use super::scrape::ParserConfig;use super::types::{AttributeSelector, Document, JsonOptions};-use crate::FirecrawlError;+use crate::{AuditMetadata, FirecrawlError};/// Uploaded file payload for the `/v2/parse` endpoint.#[derive(Debug, Clone)]@@ -132,8 +132,8 @@ pub struct ParseOptions {/// Redact personally identifiable information from returned content.#[serde(rename = "redactPII")]pub redact_pii: Option<bool>,- /// Metadata to include with SIEM logging events.- pub audit_metadata: Option<HashMap<String, String>>,+ /// User attribution to include with SIEM logging events.+ pub audit_metadata: Option<AuditMetadata>,/// Request origin identifier.pub origin: Option<String>,/// Zero data retention mode.diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex f697a0a8b..f4214c945 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -9,7 +9,7 @@ use crate::types::{Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,};-use crate::FirecrawlError;+use crate::{AuditMetadata, FirecrawlError};/// Options for scraping a URL.#[serde_with::skip_serializing_none]@@ -83,8 +83,8 @@ pub struct ScrapeOptions {#[serde(rename = "redactPII")]pub redact_pii: Option<bool>,- /// Metadata to include with SIEM logging events.- pub audit_metadata: Option<HashMap<String, String>>,+ /// User attribution to include with SIEM logging events.+ pub audit_metadata: Option<AuditMetadata>,/// Persistent browser profile for maintaining state across scrapes.pub profile: Option<ProfileConfig>,diff --git a/apps/rust-sdk/tests/audit_metadata.rs b/apps/rust-sdk/tests/audit_metadata.rsindex 55c46c242..bcfbbd1d4 100644--- a/apps/rust-sdk/tests/audit_metadata.rs+++ b/apps/rust-sdk/tests/audit_metadata.rs@@ -1,16 +1,16 @@-use std::collections::HashMap;--use firecrawl::{AgentOptions, MapOptions, ParseOptions, ScrapeOptions};+use firecrawl::{AgentOptions, AuditMetadata, MapOptions, ParseOptions, ScrapeOptions};use serde::Serialize;fn assert_audit_metadata<T: Serialize>(options: T) {let value = serde_json::to_value(options).expect("options should serialize");- assert_eq!(value["auditMetadata"]["requestId"], "req-123");+ assert_eq!(value["auditMetadata"]["username"], "[email protected]");}#[test]fn serializes_audit_metadata_across_request_options() {- let metadata = HashMap::from([("requestId".to_string(), "req-123".to_string())]);+ let metadata = AuditMetadata {+ username: "[email protected]".to_string(),+ };assert_audit_metadata(ScrapeOptions {audit_metadata: Some(metadata.clone()),diff --git a/apps/siem/azure-sentinel/DCR.json b/apps/siem/azure-sentinel/DCR.jsonindex d8e2fe29e..ad2a61002 100644--- a/apps/siem/azure-sentinel/DCR.json+++ b/apps/siem/azure-sentinel/DCR.json@@ -106,7 +106,7 @@{"streams": ["Custom-FirecrawlScrapeActivity"],"destinations": ["firecrawlWorkspace"],- "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"audit_metadata\", audit_metadata, \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)","outputStream": "Microsoft-ASimWebSessionLogs"}],diff --git a/apps/siem/azure-sentinel/azuredeploy.json b/apps/siem/azure-sentinel/azuredeploy.jsonindex 955d7a188..125a09ea8 100644--- a/apps/siem/azure-sentinel/azuredeploy.json+++ b/apps/siem/azure-sentinel/azuredeploy.json@@ -145,7 +145,7 @@{"streams": ["Custom-FirecrawlScrapeActivity"],"destinations": ["firecrawlWorkspace"],- "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"audit_metadata\", audit_metadata, \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)","outputStream": "Microsoft-ASimWebSessionLogs"}],