A free, self-hostable news aggregator…
Fix domain-wide Retry-After (#9390)
Nikolai Shcheglovf65d2760e365
diff --git a/app/Models/Feed.php b/app/Models/Feed.phpindex 3f45cd300..9dc697214 100644--- a/app/Models/Feed.php+++ b/app/Models/Feed.php@@ -605,15 +605,19 @@ class FreshRSS_Feed extends Minz_Model {Minz_Exception::ERROR);} else {- if (($retryAfter = FreshRSS_http_Util::getRetryAfter($this->url, $this->proxyParam())) > 0) {+ $url = htmlspecialchars_decode($this->url, ENT_QUOTES);+ $forceFeed = str_ends_with($url, '#force_feed');+ if ($forceFeed) {+ $url = substr($url, 0, -11);+ }+ // Retry-After is stored for the URL as requested, not as stored for the feed+ if (($retryAfter = FreshRSS_http_Util::getRetryAfter($url, $this->proxyParam())) > 0) {throw new FreshRSS_Feed_Exception('For that domain, will first retry after ' . date('c', $retryAfter) .'. ' . $this->url(includeCredentials: false), code: 503);}$simplePie = new FreshRSS_SimplePieCustom($this->attributes(), $this->curlOptions());- $url = htmlspecialchars_decode($this->url, ENT_QUOTES);- if (str_ends_with($url, '#force_feed')) {+ if ($forceFeed) {$simplePie->force_feed(true);- $url = substr($url, 0, -11);}$simplePie->set_feed_url($url);if (!$loadDetails) { //Only activates auto-discovery when adding a new feeddiff --git a/app/Utils/httpUtil.php b/app/Utils/httpUtil.phpindex 9cfb60984..b6f3d9926 100644--- a/app/Utils/httpUtil.php+++ b/app/Utils/httpUtil.php@@ -22,13 +22,16 @@ final class FreshRSS_http_Util {];/** @var array<string, string[]> $resolve_ok */private static array $resolve_ok = [];+ /** @var array<string, bool> $retry_after_domain_wide */+ private static array $retry_after_domain_wide = [];private static function getRetryAfterFile(string $url, string $proxy): string {$domain = parse_url($url, PHP_URL_HOST);if (!is_string($domain) || $domain === '') {return '';}- $domainWide = Minz_Request::serverIsPublic($domain);+ // Once per host, as serverIsPublic() may resolve it+ $domainWide = self::$retry_after_domain_wide[$domain] ??= Minz_Request::serverIsPublic($url);$port = parse_url($url, PHP_URL_PORT);if (is_int($port)) {$domain .= ':' . $port;diff --git a/tests/app/Models/FeedTest.php b/tests/app/Models/FeedTest.phpnew file mode 100644index 000000000..9668fd304--- /dev/null+++ b/tests/app/Models/FeedTest.php@@ -0,0 +1,32 @@+<?php+declare(strict_types=1);++use PHPUnit\Framework\Attributes\DataProvider;++final class FeedTest extends \PHPUnit\Framework\TestCase {++ #[DataProvider('provideStoredAndRequestedUrls')]+ public function test_load_obeysRetryAfter(string $storedUrl, string $requestedUrl): void {+ $getRetryAfterFile = new ReflectionMethod(FreshRSS_http_Util::class, 'getRetryAfterFile');+ $file = $getRetryAfterFile->invoke(null, $requestedUrl, '');+ self::assertIsString($file);+ self::assertTrue(touch($file, time() + 600));+ try {+ (new FreshRSS_Feed($storedUrl, validate: false))->load();+ self::fail('The feed was loaded during its Retry-After');+ } catch (FreshRSS_Feed_Exception $e) {+ self::assertStringContainsString('will first retry after', $e->getMessage());+ } finally {+ unlink($file);+ }+ }++ /** @return array<string,array{string,string}> */+ public static function provideStoredAndRequestedUrls(): array {+ // On a local network, where Retry-After is per URL+ return [+ 'HTML-encoded' => ['http://192.168.1.2/?action=display&bridge=Example', 'http://192.168.1.2/?action=display&bridge=Example'],+ 'force_feed' => ['http://192.168.1.2/feed#force_feed', 'http://192.168.1.2/feed'],+ ];+ }+}diff --git a/tests/app/Utils/httpUtilTest.php b/tests/app/Utils/httpUtilTest.phpindex 2ce6fe42d..930247116 100644--- a/tests/app/Utils/httpUtilTest.php+++ b/tests/app/Utils/httpUtilTest.php@@ -13,6 +13,25 @@ class httpUtilTest extends \PHPUnit\Framework\TestCase {self::assertEquals($expected, FreshRSS_http_Util::compareUrlIgnoringHttps($url1, $url2) === 0);}+ #[DataProvider('provideUrlsForRetryAfter')]+ public function test_getRetryAfterFile(string $url1, string $url2, bool $sameFile): void {+ $getRetryAfterFile = new ReflectionMethod(FreshRSS_http_Util::class, 'getRetryAfterFile');+ self::assertSame($sameFile, $getRetryAfterFile->invoke(null, $url1, '') === $getRetryAfterFile->invoke(null, $url2, ''));+ }++ /** @return array<string,array{string,string,bool}> */+ public static function provideUrlsForRetryAfter(): array {+ return [+ // A public server waits as a whole, per port+ 'public server' => ['https://198.51.100.7/feed1', 'https://198.51.100.7/feed2?a=1&b=2', true],+ 'public server, other port' => ['https://198.51.100.7/feed', 'https://198.51.100.7:8443/feed', false],+ // A server on a local network waits URL by URL+ 'local IP address' => ['http://192.168.1.2/feed1', 'http://192.168.1.2/feed2', false],+ 'local domain' => ['http://rss-bridge.lan/?bridge=A', 'http://rss-bridge.lan/?bridge=B', false],+ 'local, same URL' => ['http://192.168.1.2/feed', 'http://192.168.1.2/feed', true],+ ];+ }+#[DataProvider('provideCidrRanges')]public function test_checkCIDR(string $ip, string $range, bool $expected): void {$checkCIDR = new ReflectionMethod(FreshRSS_http_Util::class, 'checkCIDR');