Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(.net-sdk): address cubic review comments and fix CI build
Devin AI0e5c21c75fd2
diff --git a/apps/.net-sdk/Firecrawl/Firecrawl.csproj b/apps/.net-sdk/Firecrawl/Firecrawl.csprojindex 4d7f2ea8b..9fa985ec2 100644--- a/apps/.net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/.net-sdk/Firecrawl/Firecrawl.csproj@@ -20,12 +20,16 @@<PackageReadmeFile>README.md</PackageReadmeFile></PropertyGroup>+ <ItemGroup>+ <InternalsVisibleTo Include="Firecrawl.Tests" />+ </ItemGroup>+<ItemGroup><PackageReference Include="System.Text.Json" Version="9.0.3" /></ItemGroup><ItemGroup>- <None Include="..\README.md" Pack="true" PackagePath="\" />+ <None Include="../README.md" Pack="true" PackagePath="/" /></ItemGroup></Project>diff --git a/apps/.net-sdk/Firecrawl/FirecrawlClient.cs b/apps/.net-sdk/Firecrawl/FirecrawlClient.csindex fa2cc920a..fb53d7dc7 100644--- a/apps/.net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/.net-sdk/Firecrawl/FirecrawlClient.cs@@ -384,6 +384,7 @@ public class FirecrawlClientcurrent = nextPage;}+ job.Next = null;return job;}@@ -405,6 +406,7 @@ public class FirecrawlClientcurrent = nextPage;}+ job.Next = null;return job;}diff --git a/apps/.net-sdk/Firecrawl/FirecrawlHttpClient.cs b/apps/.net-sdk/Firecrawl/FirecrawlHttpClient.csindex b8f1cae42..5fe0b694e 100644--- a/apps/.net-sdk/Firecrawl/FirecrawlHttpClient.cs+++ b/apps/.net-sdk/Firecrawl/FirecrawlHttpClient.cs@@ -86,6 +86,16 @@ internal class FirecrawlHttpClientinternal async Task<T> GetAbsoluteAsync<T>(string absoluteUrl, CancellationToken cancellationToken = default){+ // Validate that the pagination URL belongs to the same host to prevent API key exfiltration+ var targetUri = new Uri(absoluteUrl);+ var baseUri = new Uri(_baseUrl);+ if (!string.Equals(targetUri.Host, baseUri.Host, StringComparison.OrdinalIgnoreCase))+ {+ throw new FirecrawlException(+ $"Pagination URL host '{targetUri.Host}' does not match API base URL host '{baseUri.Host}'. " ++ "Refusing to send credentials to a different host.");+ }+var request = new HttpRequestMessage(HttpMethod.Get, absoluteUrl);request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));@@ -156,7 +166,7 @@ internal class FirecrawlHttpClient{throw;}- catch (Exception ex)+ catch (HttpRequestException ex){if (attempt < _maxRetries){diff --git a/apps/.net-sdk/Firecrawl/Models/Document.cs b/apps/.net-sdk/Firecrawl/Models/Document.csindex c16b687f3..754ad665a 100644--- a/apps/.net-sdk/Firecrawl/Models/Document.cs+++ b/apps/.net-sdk/Firecrawl/Models/Document.cs@@ -26,10 +26,10 @@ public class Documentpublic Dictionary<string, object>? Metadata { get; set; }[JsonPropertyName("links")]- public List<object>? Links { get; set; }+ public List<string>? Links { get; set; }[JsonPropertyName("images")]- public List<object>? Images { get; set; }+ public List<string>? Images { get; set; }[JsonPropertyName("screenshot")]public string? Screenshot { get; set; }diff --git a/apps/.net-sdk/Firecrawl/Models/MapOptions.cs b/apps/.net-sdk/Firecrawl/Models/MapOptions.csindex 25d232022..912c628b7 100644--- a/apps/.net-sdk/Firecrawl/Models/MapOptions.cs+++ b/apps/.net-sdk/Firecrawl/Models/MapOptions.cs@@ -38,4 +38,8 @@ public class MapOptions[JsonPropertyName("location")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public LocationConfig? Location { get; set; }++ [JsonPropertyName("ignoreCache")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public bool? IgnoreCache { get; set; }}diff --git a/apps/.net-sdk/Firecrawl/Models/SearchData.cs b/apps/.net-sdk/Firecrawl/Models/SearchData.csindex 8675508d1..70fa4cb8e 100644--- a/apps/.net-sdk/Firecrawl/Models/SearchData.cs+++ b/apps/.net-sdk/Firecrawl/Models/SearchData.cs@@ -2,17 +2,50 @@ using System.Text.Json.Serialization;namespace Firecrawl.Models;+/// <summary>+/// Represents a single search hit with URL, title, and snippet.+/// </summary>+public class SearchHit+{+ [JsonPropertyName("url")]+ public string? Url { get; set; }++ [JsonPropertyName("title")]+ public string? Title { get; set; }++ [JsonPropertyName("description")]+ public string? Description { get; set; }++ [JsonPropertyName("markdown")]+ public string? Markdown { get; set; }++ [JsonPropertyName("html")]+ public string? Html { get; set; }++ [JsonPropertyName("rawHtml")]+ public string? RawHtml { get; set; }++ [JsonPropertyName("metadata")]+ public Dictionary<string, object>? Metadata { get; set; }++ [JsonPropertyName("links")]+ public List<string>? Links { get; set; }++ [JsonPropertyName("screenshot")]+ public string? Screenshot { get; set; }+}+/// <summary>/// Web search results./// </summary>public class SearchData{[JsonPropertyName("web")]- public List<Document>? Web { get; set; }+ public List<SearchHit>? Web { get; set; }[JsonPropertyName("news")]- public List<Document>? News { get; set; }+ public List<SearchHit>? News { get; set; }[JsonPropertyName("images")]- public List<object>? Images { get; set; }+ public List<string>? Images { get; set; }}diff --git a/apps/.net-sdk/Firecrawl/Models/SearchOptions.cs b/apps/.net-sdk/Firecrawl/Models/SearchOptions.csindex 825528d21..2cb411c51 100644--- a/apps/.net-sdk/Firecrawl/Models/SearchOptions.cs+++ b/apps/.net-sdk/Firecrawl/Models/SearchOptions.cs@@ -42,4 +42,12 @@ public class SearchOptions[JsonPropertyName("integration")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public string? Integration { get; set; }++ [JsonPropertyName("country")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public string? Country { get; set; }++ [JsonPropertyName("enterprise")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public bool? Enterprise { get; set; }}diff --git a/apps/.net-sdk/Firecrawl/Models/WebhookConfig.cs b/apps/.net-sdk/Firecrawl/Models/WebhookConfig.csindex 8c9721fb7..4cbe8a047 100644--- a/apps/.net-sdk/Firecrawl/Models/WebhookConfig.cs+++ b/apps/.net-sdk/Firecrawl/Models/WebhookConfig.cs@@ -8,7 +8,7 @@ namespace Firecrawl.Models;public class WebhookConfig{[JsonPropertyName("url")]- public string Url { get; set; } = string.Empty;+ public required string Url { get; set; }[JsonPropertyName("headers")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]diff --git a/apps/.net-sdk/README.md b/apps/.net-sdk/README.mdindex 3bace5d2b..65dfdafba 100644--- a/apps/.net-sdk/README.md+++ b/apps/.net-sdk/README.md@@ -28,7 +28,7 @@ Console.WriteLine(doc.Markdown);### API Key-The API key can be provided in three ways (in order of precedence):+The API key can be provided in two ways (in order of precedence):1. Constructor parameter: `new FirecrawlClient("fc-your-api-key")`2. Environment variable: `FIRECRAWL_API_KEY`