Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

fix(funasr): sign Aliyun intranet URLs at target host

LauraGPTb47aafb6acb4
diff --git a/config/env.transcription.example b/config/env.transcription.example
index 92418fd7..72e4dbd4 100644
--- a/config/env.transcription.example
+++ b/config/env.transcription.example
@@ -181,11 +181,14 @@ TRANSCRIPTION_MODEL=gpt-4o-transcribe-diarize
#
# --- S3 Storage (Required for FunASR) ---
# FunASR requires audio files to be available through pre-signed S3 URLs.
-# S3_ENDPOINT_URL must be publicly reachable by Alibaba Cloud DashScope; do not
-# use a VPC-only endpoint here. Speakr creates signed URLs per transcription task
-# and does not persist them in the database.
+# S3_ENDPOINT_URL is the endpoint Speakr uses for storage and for signed URLs by
+# default. Speakr creates signed URLs per transcription task and does not persist
+# them in the database.
# S3_BUCKET_NAME=your-bucket
# S3_ENDPOINT_URL=https://oss-cn-shanghai.aliyuncs.com
+# Optional: use an Aliyun internal endpoint only for URLs sent to DashScope.
+# The URL is signed directly for this host, so do not use a bucket-qualified URL.
+# S3_INTRANET_ENDPOINT_URL=https://oss-cn-shanghai-internal.aliyuncs.com
# S3_REGION=cn-shanghai
# S3_ACCESS_KEY_ID=your-access-key-id
# S3_SECRET_ACCESS_KEY=your-secret-access-key
diff --git a/src/services/storage/s3.py b/src/services/storage/s3.py
index 6b25fd53..4a6c371f 100644
--- a/src/services/storage/s3.py
+++ b/src/services/storage/s3.py
@@ -6,11 +6,19 @@ import os
import tempfile
from pathlib import Path
from typing import BinaryIO, Optional
+from urllib.parse import urlparse
from .interfaces import MaterializedFile, ObjectStat, StorageLocator, StoredObject
from .locator import build_s3_locator
+def _is_aliyun_oss_endpoint(endpoint_url: Optional[str]) -> bool:
+ if not endpoint_url:
+ return False
+ hostname = urlparse(endpoint_url).hostname or ''
+ return hostname.endswith('.aliyuncs.com')
+
+
class S3StorageBackend:
"""S3 storage backend with lazy boto3 initialization."""
@@ -28,24 +36,22 @@ class S3StorageBackend:
self.verify_ssl = verify_ssl
self._client = None
- def _get_client(self):
- if self._client is not None:
- return self._client
-
+ def _create_client(self, endpoint_url: Optional[str] = None):
try:
import boto3
from botocore.config import Config
except Exception as exc:
raise RuntimeError('S3 backend requires boto3 and botocore installed') from exc
+ resolved_endpoint = self.endpoint_url if endpoint_url is None else endpoint_url
client_kwargs = {
'service_name': 's3',
'verify': self.verify_ssl,
}
if self.region:
client_kwargs['region_name'] = self.region
- if self.endpoint_url:
- client_kwargs['endpoint_url'] = self.endpoint_url
+ if resolved_endpoint:
+ client_kwargs['endpoint_url'] = resolved_endpoint
if self.access_key_id:
client_kwargs['aws_access_key_id'] = self.access_key_id
if self.secret_access_key:
@@ -53,10 +59,19 @@ class S3StorageBackend:
if self.session_token:
client_kwargs['aws_session_token'] = self.session_token
- addressing_style = 'path' if self.use_path_style else 'auto'
+ if _is_aliyun_oss_endpoint(resolved_endpoint):
+ addressing_style = 'virtual'
+ else:
+ addressing_style = 'path' if self.use_path_style else 'auto'
client_kwargs['config'] = Config(signature_version='s3v4', s3={'addressing_style': addressing_style})
- self._client = boto3.client(**client_kwargs)
+ return boto3.client(**client_kwargs)
+
+ def _get_client(self):
+ if self._client is not None:
+ return self._client
+
+ self._client = self._create_client()
return self._client
def _bucket_key(self, locator: StorageLocator):
@@ -92,10 +107,15 @@ class S3StorageBackend:
extra['ContentType'] = content_type
if metadata:
extra['Metadata'] = metadata
- # Use put_object (single PUT) instead of upload_file (multipart with chunked encoding)
- # Aliyun OSS rejects aws-chunked encoding used by boto3's multipart upload
- with open(local_path, 'rb') as body:
- client.put_object(Bucket=self.bucket, Key=key, Body=body, **extra)
+ if _is_aliyun_oss_endpoint(self.endpoint_url):
+ # Aliyun OSS rejects the aws-chunked encoding used by boto3's
+ # managed multipart uploader.
+ with open(local_path, 'rb') as body:
+ client.put_object(Bucket=self.bucket, Key=key, Body=body, **extra)
+ elif extra:
+ client.upload_file(local_path, self.bucket, key, ExtraArgs=extra)
+ else:
+ client.upload_file(local_path, self.bucket, key)
if delete_source:
try:
os.remove(local_path)
@@ -145,8 +165,12 @@ class S3StorageBackend:
return MaterializedFile(local_path=tmp_path, cleanup_required=True)
def presign_get_url(self, locator: StorageLocator, expires_seconds: int, response_content_type: Optional[str] = None,
- response_content_disposition: Optional[str] = None) -> str:
- client = self._get_client()
+ response_content_disposition: Optional[str] = None,
+ endpoint_url: Optional[str] = None) -> str:
+ if endpoint_url and endpoint_url != self.endpoint_url:
+ client = self._create_client(endpoint_url)
+ else:
+ client = self._get_client()
bucket, key = self._bucket_key(locator)
params = {'Bucket': bucket, 'Key': key}
if response_content_type:
diff --git a/src/services/transcription/connectors/alibaba_funasr.py b/src/services/transcription/connectors/alibaba_funasr.py
index a0e38a17..c7079661 100644
--- a/src/services/transcription/connectors/alibaba_funasr.py
+++ b/src/services/transcription/connectors/alibaba_funasr.py
@@ -590,11 +590,11 @@ def prepare_funasr_file_url(recording) -> tuple:
content_type=recording.mime_type or 'application/octet-stream',
)
- url = storage.s3.presign_get_url(s3_locator, expires_seconds=86400)
-
- if S3_INTRANET_ENDPOINT_URL:
- from src.config.app_config import S3_ENDPOINT_URL
- url = url.replace(S3_ENDPOINT_URL, S3_INTRANET_ENDPOINT_URL)
+ url = storage.s3.presign_get_url(
+ s3_locator,
+ expires_seconds=86400,
+ endpoint_url=S3_INTRANET_ENDPOINT_URL,
+ )
current_app.logger.info(f"FunASR URL ready for recording {recording.id}")
return True, [url]
diff --git a/tests/test_cov_storage_s3.py b/tests/test_cov_storage_s3.py
index 347ac582..acf87f61 100644
--- a/tests/test_cov_storage_s3.py
+++ b/tests/test_cov_storage_s3.py
@@ -6,6 +6,7 @@ All tests run fully offline using a mocked boto3 client. No real network/AWS.
import os
import tempfile
from datetime import datetime
+from urllib.parse import parse_qs, urlparse
from unittest.mock import MagicMock, patch
import pytest
@@ -190,6 +191,36 @@ def test_upload_local_file_without_extra_args():
fake.upload_file.assert_called_once_with('/tmp/src.mp3', 'mybucket', 'audio/z.mp3')
+def test_upload_local_file_aliyun_uses_single_put():
+ backend = make_backend(endpoint_url='https://oss-cn-shanghai.aliyuncs.com')
+ fake = MagicMock()
+ fake.head_object.return_value = make_head_response(size=3)
+ fd, tmp_path = tempfile.mkstemp(prefix='speakr_test_aliyun_')
+ os.write(fd, b'abc')
+ os.close(fd)
+
+ try:
+ with patch_client(backend, fake):
+ result = backend.upload_local_file(
+ tmp_path,
+ 'audio/z.mp3',
+ content_type='audio/mpeg',
+ metadata={'a': 'b'},
+ )
+ finally:
+ os.remove(tmp_path)
+
+ fake.upload_file.assert_not_called()
+ kwargs = fake.put_object.call_args.kwargs
+ assert kwargs['Bucket'] == 'mybucket'
+ assert kwargs['Key'] == 'audio/z.mp3'
+ assert kwargs['ContentType'] == 'audio/mpeg'
+ assert kwargs['Metadata'] == {'a': 'b'}
+ assert kwargs['Body'].name == tmp_path
+ assert kwargs['Body'].closed is True
+ assert result.size == 3
+
+
def test_upload_local_file_delete_source_removes_file():
backend = make_backend()
fake = MagicMock()
@@ -364,6 +395,33 @@ def test_presign_get_url_with_response_overrides():
)
+def test_presign_get_url_endpoint_override_signs_the_target_host():
+ backend = make_backend(
+ region='cn-shanghai',
+ endpoint_url='https://oss-cn-shanghai.aliyuncs.com',
+ access_key_id='AKIDEXAMPLE',
+ secret_access_key='secret',
+ )
+
+ public_url = backend.presign_get_url(s3_locator(key='audio/p.mp3'), 600)
+ intranet_url = backend.presign_get_url(
+ s3_locator(key='audio/p.mp3'),
+ 600,
+ endpoint_url='https://oss-cn-shanghai-internal.aliyuncs.com',
+ )
+
+ public = urlparse(public_url)
+ intranet = urlparse(intranet_url)
+ public_query = parse_qs(public.query)
+ intranet_query = parse_qs(intranet.query)
+
+ assert public.hostname == 'mybucket.oss-cn-shanghai.aliyuncs.com'
+ assert intranet.hostname == 'mybucket.oss-cn-shanghai-internal.aliyuncs.com'
+ assert public_query['X-Amz-SignedHeaders'] == ['host']
+ assert intranet_query['X-Amz-SignedHeaders'] == ['host']
+ assert public_query['X-Amz-Signature'] != intranet_query['X-Amz-Signature']
+
+
# ---------------------------------------------------------------------------
# _get_client config wiring (patch boto3.client and inspect kwargs)
# ---------------------------------------------------------------------------
@@ -415,6 +473,16 @@ def test_get_client_virtual_addressing_and_minimal_kwargs():
assert config_kwargs['s3'] == {'addressing_style': 'auto'}
+def test_get_client_uses_virtual_addressing_for_aliyun_oss():
+ backend = make_backend(endpoint_url='https://oss-cn-shanghai.aliyuncs.com')
+ with patch('boto3.client', return_value=MagicMock()), patch(
+ 'botocore.config.Config', return_value=MagicMock()
+ ) as mock_config:
+ backend._get_client()
+
+ assert mock_config.call_args.kwargs['s3'] == {'addressing_style': 'virtual'}
+
+
def test_get_client_is_cached():
backend = make_backend()
fake_boto_client = MagicMock()
diff --git a/tests/test_funasr.py b/tests/test_funasr.py
index a0b63e43..f378686c 100644
--- a/tests/test_funasr.py
+++ b/tests/test_funasr.py
@@ -273,6 +273,31 @@ def test_prepare_existing_s3_audio_without_duplicate_upload():
assert signed_locator.raw == original_locator
+def test_prepare_audio_signs_directly_for_configured_intranet_endpoint():
+ original_locator = 's3://bucket/recordings/source.wav'
+ storage = _fake_storage(audio_locator=original_locator)
+ recording = SimpleNamespace(
+ id=1,
+ audio_path=original_locator,
+ original_filename='source.wav',
+ mime_type='audio/wav',
+ )
+
+ with Flask(__name__).app_context(), patch(
+ 'src.services.storage.get_storage_service', return_value=storage
+ ), patch(
+ 'src.config.app_config.S3_INTRANET_ENDPOINT_URL',
+ 'https://oss-cn-shanghai-internal.aliyuncs.com',
+ ):
+ success, urls = prepare_funasr_file_url(recording)
+
+ assert success is True
+ assert urls == [storage.s3.presign_get_url.return_value]
+ assert storage.s3.presign_get_url.call_args.kwargs['endpoint_url'] == (
+ 'https://oss-cn-shanghai-internal.aliyuncs.com'
+ )
+
+
if __name__ == '__main__':
test_transcription_request_has_no_file_urls_field()
test_transcription_request_extra_options_pass_through()