Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(ruby-sdk): address cubic review issues
Devin AI75624816aea4
diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rbindex b7920c9ed..a44df3972 100644--- a/apps/ruby-sdk/lib/firecrawl/client.rb+++ b/apps/ruby-sdk/lib/firecrawl/client.rb@@ -36,11 +36,14 @@ module Firecrawlbackoff_factor: DEFAULT_BACKOFF_FACTOR)resolved_key = api_key || ENV["FIRECRAWL_API_KEY"]- if resolved_key.nil? || resolved_key.empty?+ if resolved_key.nil? || resolved_key.strip.empty?raise FirecrawlError, "API key is required. Provide api_key: or set FIRECRAWL_API_KEY environment variable."endresolved_url = api_url || ENV["FIRECRAWL_API_URL"] || DEFAULT_API_URL+ unless resolved_url.match?(%r{\Ahttps?://}i)+ raise FirecrawlError, "API URL must be a fully qualified HTTP or HTTPS URL (got: #{resolved_url})."+ end@http = HttpClient.new(api_key: resolved_key,diff --git a/apps/ruby-sdk/lib/firecrawl/http_client.rb b/apps/ruby-sdk/lib/firecrawl/http_client.rbindex 910d671bd..574757fb2 100644--- a/apps/ruby-sdk/lib/firecrawl/http_client.rb+++ b/apps/ruby-sdk/lib/firecrawl/http_client.rb@@ -38,8 +38,13 @@ module Firecrawlend# Sends a GET request with a full URL (for following next-page cursors).+ # Only sends the Authorization header if the URL matches the configured API origin.def get_absolute(absolute_url)uri = URI(absolute_url)+ base_uri = URI(@base_url)+ unless uri.host == base_uri.host && uri.port == base_uri.port && uri.scheme == base_uri.scheme+ raise FirecrawlError, "Absolute URL origin (#{uri.scheme}://#{uri.host}:#{uri.port}) does not match API base URL origin (#{base_uri.scheme}://#{base_uri.host}:#{base_uri.port}). Refusing to send credentials."+ endrequest = Net::HTTP::Get.new(uri)request["Authorization"] = "Bearer #{@api_key}"execute_with_retry(uri, request)diff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rbindex c5bdf54f7..1e1afc2bb 100644--- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb@@ -15,6 +15,7 @@ module Firecrawldef initialize(**kwargs)FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }+ @skip_tls_verification = true if @skip_tls_verification.nil?enddef to_hdiff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex b9d33b514..c6f3a8fa4 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -20,6 +20,19 @@ class ClientTest < Minitest::Testassert_raises(Firecrawl::FirecrawlError) { Firecrawl::Client.new }end+ def test_raises_when_whitespace_only_api_key+ ENV.delete("FIRECRAWL_API_KEY")+ assert_raises(Firecrawl::FirecrawlError) { Firecrawl::Client.new(api_key: " ") }+ end++ def test_raises_when_api_url_not_http+ assert_raises(Firecrawl::FirecrawlError) { Firecrawl::Client.new(api_key: API_KEY, api_url: "ftp://bad.host") }+ end++ def test_raises_when_api_url_has_no_scheme+ assert_raises(Firecrawl::FirecrawlError) { Firecrawl::Client.new(api_key: API_KEY, api_url: "not-a-url") }+ end+def test_from_env_with_env_varENV["FIRECRAWL_API_KEY"] = "fc-env-key"client = Firecrawl::Client.from_env@@ -441,9 +454,22 @@ class ClientTest < Minitest::Testassert_equal 1000, h["waitFor"]assert_equal false, h["mobile"]assert_equal "stealth", h["proxy"]+ assert_equal true, h["skipTlsVerification"] # defaults to truerefute h.key?("timeout") # nil values should be omittedend+ def test_scrape_options_skip_tls_defaults_to_true+ opts = Firecrawl::Models::ScrapeOptions.new+ assert_equal true, opts.skip_tls_verification+ assert_equal true, opts.to_h["skipTlsVerification"]+ end++ def test_scrape_options_skip_tls_can_be_overridden_to_false+ opts = Firecrawl::Models::ScrapeOptions.new(skip_tls_verification: false)+ assert_equal false, opts.skip_tls_verification+ assert_equal false, opts.to_h["skipTlsVerification"]+ end+def test_crawl_options_to_hopts = Firecrawl::Models::CrawlOptions.new(limit: 100,@@ -504,7 +530,7 @@ class ClientTest < Minitest::Testzero_data_retention: true)h = opts.to_h- assert_equal({ "formats" => ["markdown"] }, h["options"])+ assert_equal({ "formats" => ["markdown"], "skipTlsVerification" => true }, h["options"])assert_equal 5, h["maxConcurrency"]assert_equal true, h["zeroDataRetention"]end@@ -548,6 +574,30 @@ class ClientTest < Minitest::Testassert_equal "# Page 2", job.data[1].markdownend+ def test_crawl_pagination_rejects_third_party_url+ stub_request(:post, "#{BASE_URL}/v2/crawl")+ .to_return(+ status: 200,+ body: JSON.generate(id: "crawl-leak"),+ headers: { "Content-Type" => "application/json" }+ )++ stub_request(:get, "#{BASE_URL}/v2/crawl/crawl-leak")+ .to_return(+ status: 200,+ body: JSON.generate(+ id: "crawl-leak", status: "completed", total: 1, completed: 1,+ data: [{ markdown: "# Page 1" }],+ next: "https://evil.example.com/steal?token=yes"+ ),+ headers: { "Content-Type" => "application/json" }+ )++ assert_raises(Firecrawl::FirecrawlError) do+ @client.crawl("https://example.com", nil, poll_interval: 0, timeout: 10)+ end+ end+# ================================================================# INTERACT# ================================================================