Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: resolve merge conflict in check_version_has_incremented.py (keep both dotnet and php)
Devin AI010d30c936d0
diff --cc .github/scripts/check_version_has_incremented.pyindex 2dea3b3bc,336230bdc..129a0a3ff--- a/.github/scripts/check_version_has_incremented.py+++ b/.github/scripts/check_version_has_incremented.py@@@ -18,11 -18,11 +18,15 @@@ Local version: 1.0.Published version: 0.0.0 (0.0.0 means not yet published on Maven Central)true+python .github/scripts/check_version_has_incremented.py dotnet ./apps/.net-sdk/Firecrawl firecrawl-dotnet+Local version: 1.0.0+Published version: 0.0.0 (0.0.0 means not yet published on NuGet)+true++ python .github/scripts/check_version_has_incremented.py php ./apps/php-sdk firecrawl/firecrawl-php+ Local version: 1.0.0+ Published version: 0.0.0 (0.0.0 means not yet published on Packagist)+ true-"""import jsonimport os@@@ -87,26 -87,41 +91,61 @@@ def get_maven_central_version(package_nreturn version_match.group(1).strip()return "0.0.0"+def get_csproj_version(file_path: str) -> str:+ """Extract version string from .csproj file."""+ csproj_file = Path(file_path).read_text()+ version_match = re.search(r'<Version>(.*?)</Version>', csproj_file)+ if version_match:+ return version_match.group(1).strip()+ raise RuntimeError("Unable to find version string in .csproj file.")++def get_nuget_version(package_name: str) -> str:+ """Get latest version of .NET package from NuGet."""+ url = f"https://api.nuget.org/v3-flatcontainer/{package_name.lower()}/index.json"+ response = requests.get(url)+ if response.status_code == 404:+ return "0.0.0"+ response.raise_for_status()+ versions = response.json().get('versions', [])+ if versions:+ return versions[-1].strip()+ return "0.0.0"++ def get_php_version(file_path: str) -> str:+ """Extract version string from PHP Version.php file."""+ version_file = Path(file_path).read_text()+ version_match = re.search(r"SDK_VERSION\s*=\s*['\"]([^'\"]*)['\"]", version_file)+ if version_match:+ return version_match.group(1).strip()+ raise RuntimeError("Unable to find SDK_VERSION string in Version.php.")++ def get_packagist_version(package_name: str) -> str:+ """Get latest version of PHP package from Packagist. package_name should be vendor/package."""+ url = f"https://packagist.org/packages/{package_name}.json"+ response = requests.get(url)+ if response.status_code == 404:+ return "0.0.0"+ response.raise_for_status()+ data = response.json()+ package_data = data.get("package", {})+ versions = package_data.get("versions", {})+ # Filter out dev versions and find highest stable version+ stable_versions = []+ for v in versions:+ normalized = v.lstrip("v")+ if "dev" in v or not re.match(r"^\d", normalized):+ continue+ try:+ parsed = parse_version(normalized)+ except Exception:+ continue+ if not parsed.is_prerelease:+ stable_versions.append(normalized)+ if not stable_versions:+ return "0.0.0"+ stable_versions.sort(key=lambda x: parse_version(x), reverse=True)+ return stable_versions[0]+# def get_rust_version(file_path: str) -> str:# """Extract version string from Cargo.toml."""# cargo_toml = toml.load(file_path)@@@ -146,14 -161,11 +185,19 @@@ if __name__ == "__main__"current_version = get_gradle_version(os.path.join(package_path, 'build.gradle.kts'))# Get published version from Maven Centralpublished_version = get_maven_central_version(package_name)+ elif package_type == "dotnet":+ # Get current version from .csproj file — look for any .csproj in the directory+ csproj_files = list(Path(package_path).glob('*.csproj'))+ if not csproj_files:+ raise RuntimeError(f"No .csproj file found in {package_path}")+ current_version = get_csproj_version(str(csproj_files[0]))+ # Get published version from NuGet+ published_version = get_nuget_version(package_name)+ elif package_type == "php":+ # Get current version from src/Version.php+ current_version = get_php_version(os.path.join(package_path, 'src', 'Version.php'))+ # Get published version from Packagist+ published_version = get_packagist_version(package_name)# if package_type == "rust":# # Get current version from Cargo.toml# current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))@@@ -161,7 -173,7 +205,7 @@@# published_version = get_crates_version(package_name)else:- raise ValueError("Invalid package type. Use 'python', 'js', 'java', or 'dotnet'.")- raise ValueError("Invalid package type. Use 'python', 'js', 'java', or 'php'.")++ raise ValueError("Invalid package type. Use 'python', 'js', 'java', 'dotnet', or 'php'.")# Print versions for debugging# print(f"Local version: {current_version}")diff --cc apps/api/src/scraper/WebScraper/crawler.tsindex a0e4b2069,661f1eeaf..0b2b49061--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@@ -165,7 -169,7 +169,10 @@@ export class WebCrawlermaxDiscoveryDepth: this.maxDiscoveryDepth,});sitemapLinks.forEach(link => {-- denialReasons.set(link, `This URL was not crawled because the maximum discovery depth (${this.maxDiscoveryDepth}) has been reached. Discovery depth counts how many 'hops' from the starting URL a page is. To crawl more pages, increase the maxDiscoveryDepth value in your crawl request.`);++ denialReasons.set(++ link,++ `This URL was not crawled because the maximum discovery depth (${this.maxDiscoveryDepth}) has been reached. Discovery depth counts how many 'hops' from the starting URL a page is. To crawl more pages, increase the maxDiscoveryDepth value in your crawl request.`,++ );});return { links: [], denialReasons };}@@@ -601,13 -613,13 +616,7 @@@maxAge,),...robotsSitemaps.map(x =>-- this.tryFetchSitemapLinks(-- x,-- _urlsHandler,-- abort,-- mock,-- maxAge,-- ),++ this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge),),]).then(results => results.reduce((a, x) => a + x, 0)),timeoutPromise,