Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(virustotal): accept the big-file upload host for large archives

velixioc7933e21df98
diff --git a/scripts/virustotal.py b/scripts/virustotal.py
index e3cd4454b..d7ac7952f 100644
--- a/scripts/virustotal.py
+++ b/scripts/virustotal.py
@@ -20,6 +20,10 @@ import uuid
ROOT = Path(__file__).resolve().parents[1]
API = "https://www.virustotal.com/api/v3"
+API_HOSTS = ("www.virustotal.com",)
+# Files over DIRECT_LIMIT go to a one-use URL VirusTotal hands out, and that URL is
+# on its big-file host. Only the file POST may use it; API calls stay on www.
+UPLOAD_HOSTS = ("www.virustotal.com", "bigfiles.virustotal.com")
DIRECT_LIMIT = 32 * 1024 * 1024
MAX_SIZE = 650 * 1024 * 1024
REQUEST_INTERVAL = 16 # Public API: at most four requests per minute.
@@ -35,10 +39,11 @@ class NoRedirects(HTTPRedirectHandler):
return None
-def validate_url(url: str) -> str:
+def validate_url(url: str, *, upload: bool = False) -> str:
try:
parsed = urlsplit(url)
- allowed = (parsed.scheme == "https" and parsed.hostname == "www.virustotal.com"
+ allowed = (parsed.scheme == "https"
+ and parsed.hostname in (UPLOAD_HOSTS if upload else API_HOSTS)
and parsed.port in (None, 443) and not parsed.username
and not parsed.password and not parsed.fragment)
except ValueError:
@@ -63,7 +68,7 @@ def normalize_upload_url(url: str) -> str:
url = parsed._replace(scheme="https", netloc="www.virustotal.com").geturl()
except ValueError:
raise ScanError("VirusTotal returned a malformed upload URL; refusing to send the file/key.") from None
- return validate_url(url)
+ return validate_url(url, upload=True)
def multipart(path: Path, boundary: str):
@@ -91,7 +96,7 @@ class Client:
def request(self, method: str, url: str, *, file: Path | None = None,
deadline: float | None = None) -> dict:
- validate_url(url)
+ validate_url(url, upload=file is not None)
delay = 0
if self.last_request is not None:
delay = REQUEST_INTERVAL - (time.monotonic() - self.last_request)
diff --git a/tests/test_virustotal.py b/tests/test_virustotal.py
index dd796aa6a..1b8f62e96 100644
--- a/tests/test_virustotal.py
+++ b/tests/test_virustotal.py
@@ -57,6 +57,23 @@ class VirusTotalTests(unittest.TestCase):
self.assertEqual([(r.get_method(), r.full_url) for r in requests],
[("GET", f"{vt.API}/files/upload_url"), ("POST", upload_url)])
+ def test_large_upload_may_use_the_big_file_host_virustotal_returns(self):
+ """VirusTotal's /files/upload_url answers with bigfiles.virustotal.com for
+ files over 32 MiB; refusing it made every full-repo scan fail."""
+ upload_url = "https://bigfiles.virustotal.com/_ah/upload/one-use-token/?q=1"
+ self.client.opener.open.side_effect = [response({"data": upload_url}), response({"data": {"id": "big"}})]
+ with patch.object(vt, "DIRECT_LIMIT", 1), patch.object(vt.time, "sleep"):
+ self.assertEqual(self.client.upload(self.path), "big")
+ post = self.client.opener.open.call_args_list[1].args[0]
+ self.assertEqual((post.get_method(), post.full_url), ("POST", upload_url))
+ self.assertEqual(post.get_header("X-apikey"), "private-test-key")
+
+ def test_big_file_host_is_for_the_file_upload_only(self):
+ for method in ("GET", "POST"):
+ with self.subTest(method=method), self.assertRaises(vt.ScanError):
+ self.client.request(method, "https://bigfiles.virustotal.com/api/v3/files/upload_url")
+ self.client.opener.open.assert_not_called()
+
def test_documented_http_upload_urls_are_upgraded_before_sending_credentials(self):
# VirusTotal's own /files/upload_url example uses an HTTP-form URL.
for authority in ("www.virustotal.com", "www.virustotal.com:80"):
@@ -91,7 +108,15 @@ class VirusTotalTests(unittest.TestCase):
"http://[email protected]/_ah/upload/token",
"http://www.virustotal.com/_ah/upload/token#fragment",
"http://www.virustotal.com.evil.example/_ah/upload/token",
- "https://www.virustotal.com:invalid/_ah/upload/token"):
+ "https://www.virustotal.com:invalid/_ah/upload/token",
+ "http://bigfiles.virustotal.com/_ah/upload/token",
+ "https://bigfiles.virustotal.com.evil.example/_ah/upload/token",
+ "https://evilbigfiles.virustotal.com/_ah/upload/token",
+ "https://static.virustotal.com/_ah/upload/token",
+ "https://virustotal.com/_ah/upload/token",
+ "https://[email protected]/_ah/upload/token",
+ "https://bigfiles.virustotal.com:8443/_ah/upload/token",
+ "https://bigfiles.virustotal.com/_ah/upload/token#fragment"):
with self.subTest(url=url), patch.object(vt, "DIRECT_LIMIT", 1):
self.client.opener.open.reset_mock()
self.client.opener.open.side_effect = None