Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(dot-net-sdk): add agent start, status, wait and cancel (#5259)
Erik Engervallf96202035d7e
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/AgentTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/AgentTests.csnew file mode 100644index 000000000..40ab54ecd--- /dev/null+++ b/apps/dot-net-sdk/Firecrawl.Tests/AgentTests.cs@@ -0,0 +1,212 @@+using System.Net;+using System.Text;+using System.Text.Json;+using Firecrawl.Exceptions;+using Firecrawl.Models;+using Xunit;++namespace Firecrawl.Tests;++public class AgentTests+{+ private sealed class StubHandler : HttpMessageHandler+ {+ private readonly Queue<string> _bodies;++ public StubHandler(params string[] bodies) => _bodies = new Queue<string>(bodies);++ public List<HttpRequestMessage> Requests { get; } = new();++ public List<string?> RequestBodies { get; } = new();++ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)+ {+ Requests.Add(request);+ RequestBodies.Add(request.Content is null ? null : await request.Content.ReadAsStringAsync(cancellationToken));+ return new HttpResponseMessage(HttpStatusCode.OK)+ {+ Content = new StringContent(_bodies.Dequeue(), Encoding.UTF8, "application/json"),+ };+ }+ }++ private static FirecrawlClient ClientFor(StubHandler handler) =>+ new(apiKey: "fc-test-key", apiUrl: "http://localhost", maxRetries: 0, httpClient: new HttpClient(handler));++ [Fact]+ public async Task StartAgentAsync_PostsPromptAndOptions()+ {+ var handler = new StubHandler("{\"success\":true,\"id\":\"agent-1\",\"threadId\":\"thread-1\",\"threadTurn\":1}");+ var client = ClientFor(handler);++ var response = await client.StartAgentAsync("Find the pricing plans", new AgentOptions+ {+ Urls = new List<string> { "https://example.com" },+ Schema = new Dictionary<string, object>+ {+ ["type"] = "object",+ ["properties"] = new Dictionary<string, object>+ {+ ["plans"] = new Dictionary<string, object> { ["type"] = "array" },+ },+ },+ Model = "spark-2",+ Effort = "high",+ MaxCredits = 500,+ Location = new AgentLocation { Country = "DE" },+ Exchange = new Dictionary<string, object> { ["requireApproval"] = true },+ });++ Assert.True(response.Success);+ Assert.Equal("agent-1", response.Id);+ Assert.Equal("thread-1", response.ThreadId);+ Assert.Equal(1, response.ThreadTurn);++ var request = Assert.Single(handler.Requests);+ Assert.Equal(HttpMethod.Post, request.Method);+ Assert.Equal("http://localhost/v2/agent", request.RequestUri!.ToString());+ using var body = JsonDocument.Parse(handler.RequestBodies[0]!);+ var root = body.RootElement;+ Assert.Equal("Find the pricing plans", root.GetProperty("prompt").GetString());+ Assert.Equal("https://example.com", root.GetProperty("urls")[0].GetString());+ Assert.Equal("object", root.GetProperty("schema").GetProperty("type").GetString());+ Assert.Equal("spark-2", root.GetProperty("model").GetString());+ Assert.Equal("high", root.GetProperty("effort").GetString());+ Assert.Equal(500, root.GetProperty("maxCredits").GetInt32());+ Assert.Equal("DE", root.GetProperty("location").GetProperty("country").GetString());+ Assert.True(root.GetProperty("exchange").GetProperty("requireApproval").GetBoolean());+ Assert.StartsWith("dotnet-sdk@", root.GetProperty("origin").GetString());+ Assert.False(root.TryGetProperty("threadId", out _));+ Assert.False(root.TryGetProperty("webhook", out _));+ Assert.False(root.TryGetProperty("threatProtection", out _));+ }++ [Fact]+ public async Task GetAgentStatusAsync_ReadsPendingApproval()+ {+ var handler = new StubHandler(+ "{\"success\":true,\"status\":\"processing\",\"pendingApproval\":{\"kind\":\"calls\",\"approvalId\":\"approval-1\"},\"exchange\":{\"calls\":1}}");+ var client = ClientFor(handler);++ var status = await client.GetAgentStatusAsync("agent-1");++ Assert.False(status.IsDone);+ var pending = Assert.IsType<JsonElement>(status.PendingApproval);+ Assert.Equal("approval-1", pending.GetProperty("approvalId").GetString());+ Assert.IsType<JsonElement>(status.Exchange);+ Assert.Null(status.Suggestions);+ }++ [Fact]+ public async Task StartAgentAsync_RequiresPrompt()+ {+ var client = ClientFor(new StubHandler());++ await Assert.ThrowsAsync<ArgumentNullException>(+ () => client.StartAgentAsync(null!));+ }++ [Fact]+ public async Task GetAgentStatusAsync_ReadsCompletedJob()+ {+ var handler = new StubHandler(+ "{\"success\":true,\"status\":\"completed\",\"data\":{\"plans\":[\"Free\",\"Pro\"]},\"model\":\"spark-2\",\"effort\":\"low\",\"expiresAt\":\"2026-10-20T00:00:00.000Z\",\"creditsUsed\":42,\"threadId\":\"thread-1\",\"threadTurn\":2,\"mode\":\"extract\"}");+ var client = ClientFor(handler);++ var status = await client.GetAgentStatusAsync("agent/1");++ Assert.True(status.IsDone);+ Assert.Equal("completed", status.Status);+ Assert.Equal("spark-2", status.Model);+ Assert.Equal("low", status.Effort);+ Assert.Equal(42, status.CreditsUsed);+ Assert.Equal(2, status.ThreadTurn);+ Assert.Equal("extract", status.Mode);+ var data = Assert.IsType<JsonElement>(status.Data);+ Assert.Equal("Pro", data.GetProperty("plans")[1].GetString());++ var request = Assert.Single(handler.Requests);+ Assert.Equal(HttpMethod.Get, request.Method);+ Assert.Equal("http://localhost/v2/agent/agent%2F1", request.RequestUri!.AbsoluteUri);+ }++ [Fact]+ public async Task AgentAsync_PollsUntilCompleted()+ {+ var handler = new StubHandler(+ "{\"success\":true,\"id\":\"agent-1\"}",+ "{\"success\":true,\"status\":\"processing\"}",+ "{\"success\":true,\"status\":\"completed\",\"message\":\"Done\"}");+ var client = ClientFor(handler);++ var status = await client.AgentAsync("Summarize the page", pollIntervalSec: 0);++ Assert.Equal("completed", status.Status);+ Assert.Equal("Done", status.Message);+ Assert.Equal(3, handler.Requests.Count);+ Assert.Equal(HttpMethod.Post, handler.Requests[0].Method);+ Assert.All(handler.Requests.Skip(1), request =>+ {+ Assert.Equal(HttpMethod.Get, request.Method);+ Assert.Equal("http://localhost/v2/agent/agent-1", request.RequestUri!.ToString());+ });+ }++ [Fact]+ public async Task AgentAsync_ReturnsFailedJobWithPartial()+ {+ var handler = new StubHandler(+ "{\"success\":true,\"id\":\"agent-1\"}",+ "{\"success\":true,\"status\":\"failed\",\"error\":\"Credit limit reached\",\"partial\":{\"plans\":[\"Free\"]},\"partialSchemaValid\":false,\"stopReason\":\"credit_limit_reached\"}");+ var client = ClientFor(handler);++ var status = await client.AgentAsync("Summarize the page", pollIntervalSec: 0);++ Assert.Equal("failed", status.Status);+ Assert.Equal("Credit limit reached", status.Error);+ Assert.Equal("credit_limit_reached", status.StopReason);+ Assert.False(status.PartialSchemaValid);+ Assert.Null(status.Data);+ Assert.IsType<JsonElement>(status.Partial);+ }++ [Fact]+ public async Task AgentAsync_ThrowsWhenStartReturnsNoId()+ {+ var handler = new StubHandler("{\"success\":false,\"error\":\"Agent beta is not enabled\"}");+ var client = ClientFor(handler);++ var ex = await Assert.ThrowsAsync<FirecrawlException>(+ () => client.AgentAsync("Summarize the page"));++ Assert.Equal("Agent beta is not enabled", ex.Message);+ Assert.Single(handler.Requests);+ }++ [Fact]+ public async Task AgentAsync_ThrowsJobTimeoutException()+ {+ var handler = new StubHandler("{\"success\":true,\"id\":\"agent-1\"}");+ var client = ClientFor(handler);++ var ex = await Assert.ThrowsAsync<JobTimeoutException>(+ () => client.AgentAsync("Summarize the page", timeoutSec: 0));++ Assert.Equal("agent-1", ex.JobId);+ Assert.Equal(0, ex.TimeoutSeconds);+ }++ [Fact]+ public async Task CancelAgentAsync_SendsDelete()+ {+ var handler = new StubHandler("{\"success\":true}");+ var client = ClientFor(handler);++ var response = await client.CancelAgentAsync("agent-1");++ Assert.True(response.ContainsKey("success"));+ var request = Assert.Single(handler.Requests);+ Assert.Equal(HttpMethod.Delete, request.Method);+ Assert.Equal("http://localhost/v2/agent/agent-1", request.RequestUri!.ToString());+ }+}diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex 7d0ba8e2a..9713fb2aa 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -670,6 +670,70 @@ public class FirecrawlClient$"/v2/agent{query}", cancellationToken);}+ /// <summary>+ /// Starts an async agent job and returns immediately.+ /// </summary>+ public async Task<AgentResponse> StartAgentAsync(+ string prompt,+ AgentOptions? options = null,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(prompt);++ var body = BuildBody(options);+ body["prompt"] = prompt;++ return await _http.PostAsync<AgentResponse>(+ "/v2/agent", body, cancellationToken: cancellationToken);+ }++ /// <summary>+ /// Gets the status and result of an agent job.+ /// </summary>+ public async Task<AgentStatusResponse> GetAgentStatusAsync(+ string jobId,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(jobId);++ return await _http.GetAsync<AgentStatusResponse>(+ $"/v2/agent/{Uri.EscapeDataString(jobId)}", cancellationToken);+ }++ /// <summary>+ /// Runs an agent job and waits until it completes, fails, or is cancelled (auto-polling).+ /// Check <see cref="AgentStatusResponse.Status"/> on the result.+ /// </summary>+ /// <remarks>+ /// On timeout this throws <see cref="JobTimeoutException"/> and the job keeps running; pass its+ /// JobId to <see cref="GetAgentStatusAsync"/> to keep waiting or <see cref="CancelAgentAsync"/> to stop it.+ /// </remarks>+ public async Task<AgentStatusResponse> AgentAsync(+ string prompt,+ AgentOptions? options = null,+ int pollIntervalSec = DefaultPollIntervalSec,+ int timeoutSec = DefaultJobTimeoutSec,+ CancellationToken cancellationToken = default)+ {+ var start = await StartAgentAsync(prompt, options, cancellationToken);+ return await PollAgentAsync(+ start.Id ?? throw new FirecrawlException(start.Error ?? "Agent start did not return a job ID"),+ pollIntervalSec, timeoutSec, cancellationToken);+ }++ /// <summary>+ /// Cancels a running agent job.+ /// </summary>+ public async Task<Dictionary<string, object>> CancelAgentAsync(+ string jobId,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(jobId);++ return await _http.DeleteAsync<Dictionary<string, object>>(+ $"/v2/agent/{Uri.EscapeDataString(jobId)}", cancellationToken);+ }+// ================================================================// USAGE & METRICS// ================================================================@@ -742,6 +806,28 @@ public class FirecrawlClientthrow new JobTimeoutException(jobId, timeoutSec, "Batch scrape");}+ private async Task<AgentStatusResponse> PollAgentAsync(+ string jobId,+ int pollIntervalSec,+ int timeoutSec,+ CancellationToken cancellationToken)+ {+ var deadline = DateTime.UtcNow.AddSeconds(timeoutSec);++ while (DateTime.UtcNow < deadline)+ {+ cancellationToken.ThrowIfCancellationRequested();++ var status = await GetAgentStatusAsync(jobId, cancellationToken);+ if (status.IsDone)+ return status;++ await Task.Delay(TimeSpan.FromSeconds(pollIntervalSec), cancellationToken);+ }++ throw new JobTimeoutException(jobId, timeoutSec, "Agent");+ }+private async Task<CrawlJob> PaginateCrawlAsync(CrawlJob job,CancellationToken cancellationToken)diff --git a/apps/dot-net-sdk/Firecrawl/Models/AgentModels.cs b/apps/dot-net-sdk/Firecrawl/Models/AgentModels.csnew file mode 100644index 000000000..3b0ff30d8--- /dev/null+++ b/apps/dot-net-sdk/Firecrawl/Models/AgentModels.cs@@ -0,0 +1,200 @@+using System.Text.Json.Serialization;++namespace Firecrawl.Models;++/// <summary>+/// Options for starting an agent job. The prompt is passed separately.+/// </summary>+public class AgentOptions+{+ [JsonPropertyName("urls")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public List<string>? Urls { get; set; }++ /// <summary>+ /// JSON Schema describing the structured output to return.+ /// </summary>+ [JsonPropertyName("schema")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public object? Schema { get; set; }++ [JsonPropertyName("integration")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Integration { get; set; }++ [JsonPropertyName("maxCredits")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public int? MaxCredits { get; set; }++ [JsonPropertyName("strictConstrainToURLs")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public bool? StrictConstrainToURLs { get; set; }++ /// <summary>+ /// Defaults to "spark-2". "spark-1-pro" and "spark-1-mini" are deprecated and run spark-2.+ /// </summary>+ [JsonPropertyName("model")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Model { get; set; }++ /// <summary>+ /// Reasoning effort: "low", "medium", or "high".+ /// </summary>+ [JsonPropertyName("effort")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Effort { get; set; }++ /// <summary>+ /// A webhook URL string or a <see cref="WebhookConfig"/>.+ /// </summary>+ [JsonPropertyName("webhook")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public object? Webhook { get; set; }++ [JsonPropertyName("auditMetadata")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public AuditMetadata? AuditMetadata { get; set; }++ /// <summary>+ /// Continues an existing thread. Omit to start a new one.+ /// </summary>+ [JsonPropertyName("threadId")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? ThreadId { get; set; }++ /// <summary>+ /// "extract" or "chat".+ /// </summary>+ [JsonPropertyName("mode")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Mode { get; set; }++ [JsonPropertyName("location")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public AgentLocation? Location { get; set; }++ /// <summary>+ /// Per-request threat protection override, sent as-is (for example <c>new { mode = "normal" }</c>).+ /// </summary>+ [JsonPropertyName("threatProtection")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public object? ThreatProtection { get; set; }++ /// <summary>+ /// Exchange settings (toolkits, maxCalls, requireApproval, approve, decline, onTermsRequired), sent as-is.+ /// </summary>+ [JsonPropertyName("exchange")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public object? Exchange { get; set; }+}++/// <summary>+/// Country for the whole agent run. Scrapes and browser sessions are proxied from it.+/// </summary>+public class AgentLocation+{+ /// <summary>+ /// ISO 3166-1 alpha-2 country code.+ /// </summary>+ [JsonPropertyName("country")]+ public required string Country { get; set; }+}++/// <summary>+/// Response from starting an agent job.+/// </summary>+public class AgentResponse+{+ [JsonPropertyName("success")]+ public bool Success { get; set; }++ [JsonPropertyName("id")]+ public string? Id { get; set; }++ [JsonPropertyName("error")]+ public string? Error { get; set; }++ [JsonPropertyName("threadId")]+ public string? ThreadId { get; set; }++ [JsonPropertyName("threadTurn")]+ public int? ThreadTurn { get; set; }+}++/// <summary>+/// Status and result of an agent job.+/// </summary>+public class AgentStatusResponse+{+ [JsonPropertyName("success")]+ public bool Success { get; set; }++ [JsonPropertyName("status")]+ public string? Status { get; set; }++ [JsonPropertyName("error")]+ public string? Error { get; set; }++ /// <summary>+ /// The result; only set once the job has completed.+ /// </summary>+ [JsonPropertyName("data")]+ public object? Data { get; set; }++ /// <summary>+ /// Best-effort result of a failed job.+ /// </summary>+ [JsonPropertyName("partial")]+ public object? Partial { get; set; }++ [JsonPropertyName("partialSchemaValid")]+ public bool? PartialSchemaValid { get; set; }++ [JsonPropertyName("stopReason")]+ public string? StopReason { get; set; }++ [JsonPropertyName("model")]+ public string? Model { get; set; }++ [JsonPropertyName("effort")]+ public string? Effort { get; set; }++ [JsonPropertyName("expiresAt")]+ public string? ExpiresAt { get; set; }++ [JsonPropertyName("creditsUsed")]+ public int? CreditsUsed { get; set; }++ [JsonPropertyName("threadId")]+ public string? ThreadId { get; set; }++ [JsonPropertyName("threadTurn")]+ public int? ThreadTurn { get; set; }++ [JsonPropertyName("mode")]+ public string? Mode { get; set; }++ /// <summary>+ /// Assistant text reply. Chat-mode runs answer here instead of in <see cref="Data"/>.+ /// </summary>+ [JsonPropertyName("message")]+ public string? Message { get; set; }++ [JsonPropertyName("suggestions")]+ public object? Suggestions { get; set; }++ /// <summary>+ /// Set while the job waits for an approval; it stays "processing" until approved or declined.+ /// </summary>+ [JsonPropertyName("pendingApproval")]+ public object? PendingApproval { get; set; }++ [JsonPropertyName("exchange")]+ public object? Exchange { get; set; }++ /// <summary>+ /// Returns true if the agent job has reached a terminal state.+ /// </summary>+ public bool IsDone =>+ Status == "completed" || Status == "failed" || Status == "cancelled";+}diff --git a/apps/dot-net-sdk/README.md b/apps/dot-net-sdk/README.mdindex b89e10e6d..5368aa59c 100644--- a/apps/dot-net-sdk/README.md+++ b/apps/dot-net-sdk/README.md@@ -194,6 +194,43 @@ foreach (var doc in job.Data!)}```+### Agent++Describe what you need in a prompt; the agent searches, navigates and extracts it.+Pass a JSON Schema to get structured output.++```csharp+// Run an agent job with auto-polling (waits until it completes, fails or is cancelled)+var result = await client.AgentAsync("Find the pricing plans and their monthly prices",+ new AgentOptions+ {+ Urls = new List<string> { "https://example.com" },+ Schema = new Dictionary<string, object>+ {+ ["type"] = "object",+ ["properties"] = new Dictionary<string, object>+ {+ ["plans"] = new Dictionary<string, object> { ["type"] = "array" }+ }+ }+ });++if (result.Status == "completed")+ Console.WriteLine(result.Data);+else+ Console.WriteLine($"{result.Status}: {result.Error}");++// Start an agent job without waiting+var started = await client.StartAgentAsync("Summarize the latest blog post on example.com");++// Check status later+var status = await client.GetAgentStatusAsync(started.Id!);+Console.WriteLine($"Status: {status.Status}, Credits used: {status.CreditsUsed}");++// Cancel an agent job+await client.CancelAgentAsync(started.Id!);+```+### ParseUpload and parse a local file (HTML, PDF, DOCX, etc.) through the `/v2/parse`