Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: resolve merge conflict in check_version_has_incremented.py (keep both ruby and php)
Devin AIb552f86ec1db
diff --cc .github/scripts/check_version_has_incremented.pyindex 03f321499,336230bdc..46f8aed32--- a/.github/scripts/check_version_has_incremented.py+++ b/.github/scripts/check_version_has_incremented.py@@@ -18,11 -18,11 +18,15 @@@ Local version: 1.0.Published version: 0.0.0 (0.0.0 means not yet published on Maven Central)true+python .github/scripts/check_version_has_incremented.py ruby ./apps/ruby-sdk firecrawl+Local version: 1.0.0+Published version: 0.0.0 (0.0.0 means not yet published on RubyGems)+true++ python .github/scripts/check_version_has_incremented.py php ./apps/php-sdk firecrawl/firecrawl-php+ Local version: 1.0.0+ Published version: 0.0.0 (0.0.0 means not yet published on Packagist)+ true-"""import jsonimport os@@@ -142,11 -161,11 +181,16 @@@ if __name__ == "__main__"current_version = get_gradle_version(os.path.join(package_path, 'build.gradle.kts'))# Get published version from Maven Centralpublished_version = get_maven_central_version(package_name)+ elif package_type == "ruby":+ # Get current version from lib/firecrawl/version.rb+ current_version = get_ruby_version(os.path.join(package_path, 'lib', 'firecrawl', 'version.rb'))+ # Get published version from RubyGems+ published_version = get_rubygems_version(package_name)+ elif package_type == "php":+ # Get current version from src/Version.php+ current_version = get_php_version(os.path.join(package_path, 'src', 'Version.php'))+ # Get published version from Packagist+ published_version = get_packagist_version(package_name)# if package_type == "rust":# # Get current version from Cargo.toml# current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))@@@ -154,7 -173,7 +198,7 @@@# published_version = get_crates_version(package_name)else:- raise ValueError("Invalid package type. Use 'python', 'js', 'java', or 'ruby'.")- raise ValueError("Invalid package type. Use 'python', 'js', 'java', or 'php'.")++ raise ValueError("Invalid package type. Use 'python', 'js', 'java', 'ruby', or 'php'.")# Print versions for debugging# print(f"Local version: {current_version}")diff --cc apps/api/src/scraper/WebScraper/crawler.tsindex a0e4b2069,661f1eeaf..0b2b49061--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@@ -165,7 -169,7 +169,10 @@@ export class WebCrawlermaxDiscoveryDepth: this.maxDiscoveryDepth,});sitemapLinks.forEach(link => {-- denialReasons.set(link, `This URL was not crawled because the maximum discovery depth (${this.maxDiscoveryDepth}) has been reached. Discovery depth counts how many 'hops' from the starting URL a page is. To crawl more pages, increase the maxDiscoveryDepth value in your crawl request.`);++ denialReasons.set(++ link,++ `This URL was not crawled because the maximum discovery depth (${this.maxDiscoveryDepth}) has been reached. Discovery depth counts how many 'hops' from the starting URL a page is. To crawl more pages, increase the maxDiscoveryDepth value in your crawl request.`,++ );});return { links: [], denialReasons };}@@@ -601,13 -613,13 +616,7 @@@maxAge,),...robotsSitemaps.map(x =>-- this.tryFetchSitemapLinks(-- x,-- _urlsHandler,-- abort,-- mock,-- maxAge,-- ),++ this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge),),]).then(results => results.reduce((a, x) => a + x, 0)),timeoutPromise,