Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
fix(funasr): sign Aliyun intranet URLs at target host
LauraGPTb47aafb6acb4
diff --git a/config/env.transcription.example b/config/env.transcription.exampleindex 92418fd7..72e4dbd4 100644--- a/config/env.transcription.example+++ b/config/env.transcription.example@@ -181,11 +181,14 @@ TRANSCRIPTION_MODEL=gpt-4o-transcribe-diarize## --- S3 Storage (Required for FunASR) ---# FunASR requires audio files to be available through pre-signed S3 URLs.-# S3_ENDPOINT_URL must be publicly reachable by Alibaba Cloud DashScope; do not-# use a VPC-only endpoint here. Speakr creates signed URLs per transcription task-# and does not persist them in the database.+# S3_ENDPOINT_URL is the endpoint Speakr uses for storage and for signed URLs by+# default. Speakr creates signed URLs per transcription task and does not persist+# them in the database.# S3_BUCKET_NAME=your-bucket# S3_ENDPOINT_URL=https://oss-cn-shanghai.aliyuncs.com+# Optional: use an Aliyun internal endpoint only for URLs sent to DashScope.+# The URL is signed directly for this host, so do not use a bucket-qualified URL.+# S3_INTRANET_ENDPOINT_URL=https://oss-cn-shanghai-internal.aliyuncs.com# S3_REGION=cn-shanghai# S3_ACCESS_KEY_ID=your-access-key-id# S3_SECRET_ACCESS_KEY=your-secret-access-keydiff --git a/src/services/storage/s3.py b/src/services/storage/s3.pyindex 6b25fd53..4a6c371f 100644--- a/src/services/storage/s3.py+++ b/src/services/storage/s3.py@@ -6,11 +6,19 @@ import osimport tempfilefrom pathlib import Pathfrom typing import BinaryIO, Optional+from urllib.parse import urlparsefrom .interfaces import MaterializedFile, ObjectStat, StorageLocator, StoredObjectfrom .locator import build_s3_locator+def _is_aliyun_oss_endpoint(endpoint_url: Optional[str]) -> bool:+ if not endpoint_url:+ return False+ hostname = urlparse(endpoint_url).hostname or ''+ return hostname.endswith('.aliyuncs.com')++class S3StorageBackend:"""S3 storage backend with lazy boto3 initialization."""@@ -28,24 +36,22 @@ class S3StorageBackend:self.verify_ssl = verify_sslself._client = None- def _get_client(self):- if self._client is not None:- return self._client-+ def _create_client(self, endpoint_url: Optional[str] = None):try:import boto3from botocore.config import Configexcept Exception as exc:raise RuntimeError('S3 backend requires boto3 and botocore installed') from exc+ resolved_endpoint = self.endpoint_url if endpoint_url is None else endpoint_urlclient_kwargs = {'service_name': 's3','verify': self.verify_ssl,}if self.region:client_kwargs['region_name'] = self.region- if self.endpoint_url:- client_kwargs['endpoint_url'] = self.endpoint_url+ if resolved_endpoint:+ client_kwargs['endpoint_url'] = resolved_endpointif self.access_key_id:client_kwargs['aws_access_key_id'] = self.access_key_idif self.secret_access_key:@@ -53,10 +59,19 @@ class S3StorageBackend:if self.session_token:client_kwargs['aws_session_token'] = self.session_token- addressing_style = 'path' if self.use_path_style else 'auto'+ if _is_aliyun_oss_endpoint(resolved_endpoint):+ addressing_style = 'virtual'+ else:+ addressing_style = 'path' if self.use_path_style else 'auto'client_kwargs['config'] = Config(signature_version='s3v4', s3={'addressing_style': addressing_style})- self._client = boto3.client(**client_kwargs)+ return boto3.client(**client_kwargs)++ def _get_client(self):+ if self._client is not None:+ return self._client++ self._client = self._create_client()return self._clientdef _bucket_key(self, locator: StorageLocator):@@ -92,10 +107,15 @@ class S3StorageBackend:extra['ContentType'] = content_typeif metadata:extra['Metadata'] = metadata- # Use put_object (single PUT) instead of upload_file (multipart with chunked encoding)- # Aliyun OSS rejects aws-chunked encoding used by boto3's multipart upload- with open(local_path, 'rb') as body:- client.put_object(Bucket=self.bucket, Key=key, Body=body, **extra)+ if _is_aliyun_oss_endpoint(self.endpoint_url):+ # Aliyun OSS rejects the aws-chunked encoding used by boto3's+ # managed multipart uploader.+ with open(local_path, 'rb') as body:+ client.put_object(Bucket=self.bucket, Key=key, Body=body, **extra)+ elif extra:+ client.upload_file(local_path, self.bucket, key, ExtraArgs=extra)+ else:+ client.upload_file(local_path, self.bucket, key)if delete_source:try:os.remove(local_path)@@ -145,8 +165,12 @@ class S3StorageBackend:return MaterializedFile(local_path=tmp_path, cleanup_required=True)def presign_get_url(self, locator: StorageLocator, expires_seconds: int, response_content_type: Optional[str] = None,- response_content_disposition: Optional[str] = None) -> str:- client = self._get_client()+ response_content_disposition: Optional[str] = None,+ endpoint_url: Optional[str] = None) -> str:+ if endpoint_url and endpoint_url != self.endpoint_url:+ client = self._create_client(endpoint_url)+ else:+ client = self._get_client()bucket, key = self._bucket_key(locator)params = {'Bucket': bucket, 'Key': key}if response_content_type:diff --git a/src/services/transcription/connectors/alibaba_funasr.py b/src/services/transcription/connectors/alibaba_funasr.pyindex a0e38a17..c7079661 100644--- a/src/services/transcription/connectors/alibaba_funasr.py+++ b/src/services/transcription/connectors/alibaba_funasr.py@@ -590,11 +590,11 @@ def prepare_funasr_file_url(recording) -> tuple:content_type=recording.mime_type or 'application/octet-stream',)- url = storage.s3.presign_get_url(s3_locator, expires_seconds=86400)-- if S3_INTRANET_ENDPOINT_URL:- from src.config.app_config import S3_ENDPOINT_URL- url = url.replace(S3_ENDPOINT_URL, S3_INTRANET_ENDPOINT_URL)+ url = storage.s3.presign_get_url(+ s3_locator,+ expires_seconds=86400,+ endpoint_url=S3_INTRANET_ENDPOINT_URL,+ )current_app.logger.info(f"FunASR URL ready for recording {recording.id}")return True, [url]diff --git a/tests/test_cov_storage_s3.py b/tests/test_cov_storage_s3.pyindex 347ac582..acf87f61 100644--- a/tests/test_cov_storage_s3.py+++ b/tests/test_cov_storage_s3.py@@ -6,6 +6,7 @@ All tests run fully offline using a mocked boto3 client. No real network/AWS.import osimport tempfilefrom datetime import datetime+from urllib.parse import parse_qs, urlparsefrom unittest.mock import MagicMock, patchimport pytest@@ -190,6 +191,36 @@ def test_upload_local_file_without_extra_args():fake.upload_file.assert_called_once_with('/tmp/src.mp3', 'mybucket', 'audio/z.mp3')+def test_upload_local_file_aliyun_uses_single_put():+ backend = make_backend(endpoint_url='https://oss-cn-shanghai.aliyuncs.com')+ fake = MagicMock()+ fake.head_object.return_value = make_head_response(size=3)+ fd, tmp_path = tempfile.mkstemp(prefix='speakr_test_aliyun_')+ os.write(fd, b'abc')+ os.close(fd)++ try:+ with patch_client(backend, fake):+ result = backend.upload_local_file(+ tmp_path,+ 'audio/z.mp3',+ content_type='audio/mpeg',+ metadata={'a': 'b'},+ )+ finally:+ os.remove(tmp_path)++ fake.upload_file.assert_not_called()+ kwargs = fake.put_object.call_args.kwargs+ assert kwargs['Bucket'] == 'mybucket'+ assert kwargs['Key'] == 'audio/z.mp3'+ assert kwargs['ContentType'] == 'audio/mpeg'+ assert kwargs['Metadata'] == {'a': 'b'}+ assert kwargs['Body'].name == tmp_path+ assert kwargs['Body'].closed is True+ assert result.size == 3++def test_upload_local_file_delete_source_removes_file():backend = make_backend()fake = MagicMock()@@ -364,6 +395,33 @@ def test_presign_get_url_with_response_overrides():)+def test_presign_get_url_endpoint_override_signs_the_target_host():+ backend = make_backend(+ region='cn-shanghai',+ endpoint_url='https://oss-cn-shanghai.aliyuncs.com',+ access_key_id='AKIDEXAMPLE',+ secret_access_key='secret',+ )++ public_url = backend.presign_get_url(s3_locator(key='audio/p.mp3'), 600)+ intranet_url = backend.presign_get_url(+ s3_locator(key='audio/p.mp3'),+ 600,+ endpoint_url='https://oss-cn-shanghai-internal.aliyuncs.com',+ )++ public = urlparse(public_url)+ intranet = urlparse(intranet_url)+ public_query = parse_qs(public.query)+ intranet_query = parse_qs(intranet.query)++ assert public.hostname == 'mybucket.oss-cn-shanghai.aliyuncs.com'+ assert intranet.hostname == 'mybucket.oss-cn-shanghai-internal.aliyuncs.com'+ assert public_query['X-Amz-SignedHeaders'] == ['host']+ assert intranet_query['X-Amz-SignedHeaders'] == ['host']+ assert public_query['X-Amz-Signature'] != intranet_query['X-Amz-Signature']++# ---------------------------------------------------------------------------# _get_client config wiring (patch boto3.client and inspect kwargs)# ---------------------------------------------------------------------------@@ -415,6 +473,16 @@ def test_get_client_virtual_addressing_and_minimal_kwargs():assert config_kwargs['s3'] == {'addressing_style': 'auto'}+def test_get_client_uses_virtual_addressing_for_aliyun_oss():+ backend = make_backend(endpoint_url='https://oss-cn-shanghai.aliyuncs.com')+ with patch('boto3.client', return_value=MagicMock()), patch(+ 'botocore.config.Config', return_value=MagicMock()+ ) as mock_config:+ backend._get_client()++ assert mock_config.call_args.kwargs['s3'] == {'addressing_style': 'virtual'}++def test_get_client_is_cached():backend = make_backend()fake_boto_client = MagicMock()diff --git a/tests/test_funasr.py b/tests/test_funasr.pyindex a0b63e43..f378686c 100644--- a/tests/test_funasr.py+++ b/tests/test_funasr.py@@ -273,6 +273,31 @@ def test_prepare_existing_s3_audio_without_duplicate_upload():assert signed_locator.raw == original_locator+def test_prepare_audio_signs_directly_for_configured_intranet_endpoint():+ original_locator = 's3://bucket/recordings/source.wav'+ storage = _fake_storage(audio_locator=original_locator)+ recording = SimpleNamespace(+ id=1,+ audio_path=original_locator,+ original_filename='source.wav',+ mime_type='audio/wav',+ )++ with Flask(__name__).app_context(), patch(+ 'src.services.storage.get_storage_service', return_value=storage+ ), patch(+ 'src.config.app_config.S3_INTRANET_ENDPOINT_URL',+ 'https://oss-cn-shanghai-internal.aliyuncs.com',+ ):+ success, urls = prepare_funasr_file_url(recording)++ assert success is True+ assert urls == [storage.s3.presign_get_url.return_value]+ assert storage.s3.presign_get_url.call_args.kwargs['endpoint_url'] == (+ 'https://oss-cn-shanghai-internal.aliyuncs.com'+ )++if __name__ == '__main__':test_transcription_request_has_no_file_urls_field()test_transcription_request_extra_options_pass_through()