Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: accept subtitle, caption and the rest of the common attachment formats (#9738)

Michael Hanb480b8a6490b
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 0b6924e61c8..d535131d3e6 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7097,6 +7097,14 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo
audio_preflight["prepared"] = await asyncio.to_thread(
_prepare_audio_for_llama, audio_preflight["b64"]
)
+ except _DecodedAudioTooLongError:
+ # A limit the caller can act on. Reading as "could not be decoded"
+ # answered the same upload differently depending on whether a swap
+ # happened to be running.
+ raise HTTPException(
+ status_code = 413,
+ detail = _audio_too_long_detail(),
+ ) from None
except Exception:
raise HTTPException(
status_code = 400,
@@ -7130,6 +7138,11 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo
audio_preflight["decoded"] = await asyncio.to_thread(
_decode_audio_base64, audio_preflight["b64"]
)
+ except _DecodedAudioTooLongError:
+ raise HTTPException(
+ status_code = 413,
+ detail = _audio_too_long_detail(),
+ ) from None
except Exception:
raise HTTPException(
status_code = 400,
@@ -17704,10 +17717,69 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":
tmp.write(raw)
tmp_path = tmp.name
try:
- waveform, sr = torchaudio.load(tmp_path)
+ # A small compressed file (amr/wma/opus) can hold hours of PCM, so bound
+ # the decode the same way the GGUF path does rather than materializing it
+ # and checking after.
+ try:
+ probe = torchaudio.info(tmp_path)
+ rate = int(getattr(probe, "sample_rate", 0) or 0)
+ frames = int(getattr(probe, "num_frames", 0) or 0)
+ channels = max(1, int(getattr(probe, "num_channels", 1) or 1))
+ except Exception: # noqa: BLE001 - a container info cannot read is still loadable
+ rate, frames, channels = 0, 0, 1
+ limit = rate * _MAX_AUDIO_SECONDS
+ if limit and frames > limit:
+ raise _DecodedAudioTooLongError(
+ f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+ # load() materializes every channel at the native rate, so it is only safe
+ # when info said how long the file is and that fits the ceiling. Anything
+ # else -- an unreadable probe, an unreported length, a high rate or a
+ # multichannel file -- goes to the bounded reader, which downmixes as it
+ # goes and holds mono frames only. Streaming beats refusing: the file is
+ # inside the clock, it is just too wide to hold at once.
+ if limit and frames and frames * channels <= _MAX_DECODED_SAMPLES:
+ # One frame past the cap, so a container that misreports its length
+ # is still never fully read. Both caps apply: info() is the value
+ # being distrusted here, so an understated num_frames must not let
+ # the read run to the rate-relative limit, which at 192 kHz is four
+ # times the sample ceiling. A file that fits is unaffected: its
+ # length is under both.
+ read_frames = min(limit, _MAX_DECODED_SAMPLES // channels)
+ waveform, sr = torchaudio.load(tmp_path, num_frames = read_frames + 1)
+ else:
+ import torch
+ try:
+ samples, sr = _decode_audio_mono(raw)
+ except RuntimeError:
+ # No libsndfile, no PyAV, no librosa. This function used to call
+ # torchaudio.load() outright, so an environment carrying only
+ # torchaudio could decode audio, and an install predating PyAV
+ # as a base requirement must not lose that on upgrade.
+ #
+ # It cannot simply call load() again, though. What routes an
+ # install here is torchaudio 2.9 dropping info(), and 2.9's
+ # load() is an alias for load_with_torchcodec, which decodes the
+ # whole file with get_all_samples() and only then slices to
+ # num_frames -- so the argument bounds the return value and not
+ # the allocation, and a 25 MB Opus upload holding hours of audio
+ # exhausts the backend before any check below runs. Ask
+ # torchcodec what the file is first, and read only a bounded
+ # range of it.
+ waveform, sr = _decode_audio_with_torchcodec(tmp_path)
+ else:
+ waveform = torch.from_numpy(samples).unsqueeze(0)
finally:
os.unlink(tmp_path)
+ # Backstop for a container that reported neither rate nor length.
+ if (sr > 0 and waveform.shape[-1] > sr * _MAX_AUDIO_SECONDS) or (
+ waveform.shape[-1] * waveform.shape[0] > _MAX_DECODED_SAMPLES
+ ):
+ raise _DecodedAudioTooLongError(
+ f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+
if waveform.shape[0] > 1:
waveform = waveform.mean(dim = 0, keepdim = True)
@@ -17728,19 +17800,282 @@ _MAX_AUDIO_B64_CHARS = STT_AUDIO_B64_MAX_CHARS
# Flooring instead refused a file of exactly the size the composer allows.
_MAX_VIDEO_B64_CHARS = 4 * math.ceil((64 * 1024 * 1024) / 3)
_MAX_AUDIO_SECONDS = 30 * 60
+# The duration cap alone is rate-relative, so a high-rate container retains far
+# more memory for the same 30 minutes: at 48 kHz that is 86M float32 samples,
+# and np.concatenate doubles it. 48 kHz covers ordinary uploads, so this ceiling
+# only refuses rates above it, and never before the duration cap bites.
+_MAX_DECODED_SAMPLES = 48_000 * _MAX_AUDIO_SECONDS
+# One streamed block, every channel counted. 4 MB of float32, and the reader
+# holds a second copy of it, so this is the transient a decode may reach for
+# regardless of how many channels the container declares.
+_MAX_DECODE_BLOCK_SAMPLES = 1 << 20
+# How much of a resample is in flight at once, counted on the side that reads
+# the most samples. A few MB of float64, whatever the recording's length.
+_RESAMPLE_SLICE_SAMPLES = 1 << 18
_WAV_HEADER_BYTES = 44
_MIN_TRANSCODE_AUDIO_SAMPLE_RATE = 8000
+class _DecodedAudioTooLongError(ValueError):
+ """Decoded audio crossed the duration cap before it could be buffered."""
+
+
+def _audio_too_long_detail() -> str:
+ """The one answer an overlong upload gets, whichever path found it out."""
+ return f"Audio is too long (max {_MAX_AUDIO_SECONDS // 60} minutes)."
+
+
+def _id3_tag_length(raw: bytes) -> int:
+ """Bytes an ID3v2 tag occupies before the first MPEG frame.
+
+ An ID3v2 tag can prefix any MPEG audio layer (and occasionally AAC), so it
+ is skipped before anything reads a frame header. The four size bytes are
+ synchsafe: their top bit must be clear.
+ """
+ if len(raw) < 10 or raw[:3] != b"ID3" or any(byte & 0x80 for byte in raw[6:10]):
+ return 0
+ tag_size = 0
+ for byte in raw[6:10]:
+ tag_size = (tag_size << 7) | byte
+ length = 10 + tag_size
+ # ID3v2.4's optional footer is not included in the stored tag size.
+ if raw[3] == 4 and raw[5] & 0x10:
+ length += 10
+ return length
+
+
def _sniff_audio_container(raw: bytes) -> Optional[str]:
"""Return 'wav' or 'mp3' if the bytes are a container llama-server accepts
directly (so we can forward them untouched), else None (needs transcoding)."""
if len(raw) >= 12 and raw[:4] == b"RIFF" and raw[8:12] == b"WAVE":
return "wav"
- # mp3: ID3 tag, or an MPEG audio frame sync (no other accepted format leads
- # with 0xFF, so the simple sync check doesn't collide).
- if raw[:3] == b"ID3" or (len(raw) >= 2 and raw[0] == 0xFF and (raw[1] & 0xE0) == 0xE0):
- return "mp3"
+
+ frame_offset = _id3_tag_length(raw)
+
+ # MPEG audio frame header: the 11-bit sync is followed by version and layer
+ # bits. Layer III is binary 01; Layer II (MP2) is 10, while ADTS AAC uses
+ # the reserved MPEG layer value 00. Only Layer III is safe to pass as mp3.
+ if len(raw) >= frame_offset + 2:
+ first, second = raw[frame_offset : frame_offset + 2]
+ version = (second >> 3) & 0x03
+ layer = (second >> 1) & 0x03
+ is_mpeg_sync = first == 0xFF and (second & 0xE0) == 0xE0
+ if is_mpeg_sync and version != 0x01 and layer == 0x01:
+ return "mp3"
+
+ # ID3-only or malformed MPEG data must go through the decoder too; treating
+ # the tag itself as proof of MP3 would reintroduce the MP2/AAC collision.
+ return None
+
+
+# Layer III bitrates in kbps by MPEG version bits, and the sample rates that go
+# with them. Index 0 is "free" and index 15 is invalid; both read as 0 here.
+_MPEG_LAYER3_BITRATES = {
+ 3: (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0),
+ 2: (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0),
+ 0: (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0),
+}
+_MPEG_SAMPLE_RATES = {
+ 3: (44_100, 48_000, 32_000, 0),
+ 2: (22_050, 24_000, 16_000, 0),
+ 0: (11_025, 12_000, 8_000, 0),
+}
+
+
+def _decode_audio_with_torchcodec(path: str) -> "tuple[Any, int]":
+ """Read a bounded range of an audio file with torchcodec, or refuse.
+
+ The last resort for an install carrying torchaudio and no other decoder.
+ torchcodec is what torchaudio 2.9's own load() decodes through, so it is
+ present wherever that load() would have worked, and its metadata answers
+ the rate and duration without decoding anything.
+ """
+ try:
+ from torchcodec.decoders import AudioDecoder
+ except Exception as e: # noqa: BLE001 - no bounded reader, so no read
+ raise RuntimeError(
+ "this audio format could not be decoded; run `unsloth studio update` "
+ "to install PyAV or convert it to wav, mp3, ogg or flac"
+ ) from e
+
+ decoder = AudioDecoder(path)
+ metadata = decoder.metadata
+ rate = int(getattr(metadata, "sample_rate", 0) or 0)
+ if rate <= 0:
+ raise RuntimeError(
+ "this audio file does not report a sample rate, so it cannot be "
+ "decoded within the size limit; convert it to wav or mp3"
+ )
+ duration = float(getattr(metadata, "duration_seconds", 0.0) or 0.0)
+ if duration > _MAX_AUDIO_SECONDS:
+ raise _DecodedAudioTooLongError(
+ f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+ # Both ceilings, and a second past the clock so a container understating its
+ # own duration is still cut rather than believed.
+ channels = max(1, int(getattr(metadata, "num_channels", 1) or 1))
+ seconds = min(_MAX_AUDIO_SECONDS + 1, _MAX_DECODED_SAMPLES / (rate * channels) + 1)
+ samples = decoder.get_samples_played_in_range(0.0, seconds)
+ return samples.data, rate
+
+
+_WAVE_FORMAT_PCM = 0x0001
+_WAVE_FORMAT_IEEE_FLOAT = 0x0003
+_WAVE_FORMAT_EXTENSIBLE = 0xFFFE
+
+
+def _wav_format_is_pcm(raw: bytes, body: int, size: int, format_tag: int) -> bool:
+ """Whether a fmt chunk describes uncompressed samples.
+
+ WAVE_FORMAT_EXTENSIBLE carries the real tag in the first two bytes of its
+ SubFormat GUID, 24 bytes into the extension. A file that claims the tag
+ without carrying the extension has not said what it holds.
+ """
+ if format_tag in (_WAVE_FORMAT_PCM, _WAVE_FORMAT_IEEE_FLOAT):
+ return True
+ if format_tag != _WAVE_FORMAT_EXTENSIBLE or size < 40 or body + 26 > len(raw):
+ return False
+ sub_format = int.from_bytes(raw[body + 24 : body + 26], "little")
+ return sub_format in (_WAVE_FORMAT_PCM, _WAVE_FORMAT_IEEE_FLOAT)
+
+
+def _wav_seconds(raw: bytes) -> Optional[float]:
+ """Seconds of PCM a RIFF/WAVE header describes, or None if it cannot say.
+
+ Header arithmetic only: the point is to bound a file that is forwarded
+ without ever being decoded here.
+ """
+ if len(raw) < 12 or raw[:4] != b"RIFF" or raw[8:12] != b"WAVE":
+ return None
+ byte_rate = 0
+ offset = 12
+ while offset + 8 <= len(raw):
+ chunk = raw[offset : offset + 4]
+ size = int.from_bytes(raw[offset + 4 : offset + 8], "little")
+ body = offset + 8
+ if chunk == b"fmt " and size >= 16 and body + 16 <= len(raw):
+ format_tag = int.from_bytes(raw[body : body + 2], "little")
+ channels = int.from_bytes(raw[body + 2 : body + 4], "little")
+ sample_rate = int.from_bytes(raw[body + 4 : body + 8], "little")
+ declared_rate = int.from_bytes(raw[body + 8 : body + 12], "little")
+ block_align = int.from_bytes(raw[body + 12 : body + 14], "little")
+ bits = int.from_bytes(raw[body + 14 : body + 16], "little")
+ if not _wav_format_is_pcm(raw, body, size, format_tag):
+ # Only WAVE_FORMAT_PCM fixes nBlockAlign as one sample frame and
+ # nAvgBytesPerSec as rate * blockAlign; for every other tag both
+ # are the codec's business (mmeapi WAVEFORMATEX). An IMA ADPCM
+ # block holds ~505 frames, so PCM arithmetic overstates its rate
+ # 505-fold and read three hours as twenty-one seconds. Nothing
+ # here can check a codec's own header, so say so and let the
+ # bounded decoder have it.
+ return None
+ # Among the PCM fields nAvgBytesPerSec is the redundant one, so it
+ # is the one that can be moved alone. Multiplied by ten thousand it
+ # made half an hour read as a quarter of a second, and a quarter of
+ # a second is forwarded untouched. Recompute it, and take the
+ # declaration only where it agrees.
+ computed = sample_rate * (block_align or channels * (bits // 8))
+ byte_rate = computed if computed > 0 and declared_rate != computed else declared_rate
+ elif chunk == b"data":
+ if byte_rate <= 0:
+ return None
+ # A streaming writer can declare a size the file does not carry, so
+ # the smaller of the two is what is really there.
+ carried = len(raw) - body
+ return (min(size, carried) if size else carried) / byte_rate
+ # Chunks are padded to an even length, and the padding is not counted.
+ offset = body + size + (size & 1)
+ return None
+
+
+def _mp3_trailer_length(raw: bytes, offset: int) -> int:
+ """Bytes the metadata trailer at `offset` occupies, or 0 if there is none.
+
+ Only the trailers whose length is written down are read, because the length
+ is the whole point: a trailer that is merely recognised by its first three
+ bytes proves nothing about what follows it. `cat one.mp3 two.mp3` leaves the
+ first file's 128-byte ID3v1 tag in the middle of the result, and matching on
+ the magic alone accepted that as the end of the recording and reported the
+ first file's duration for both. Lyrics3 has no fixed-size header, so it is
+ left unrecognised and the file gets decoded rather than measured.
+ """
+ tail = raw[offset:]
+ if tail[:3] == b"TAG":
+ # ID3v1 and ID3v1.1 are both exactly 128 bytes.
+ return 128 if len(tail) >= 128 else 0
+ if tail[:3] == b"ID3":
+ return _id3_tag_length(tail)
+ if tail[:8] == b"APETAGEX" and len(tail) >= 32:
+ # The size field covers the item data and the 32-byte footer but never
+ # the header. An APEv2 header is optional, and an APEv1 tag has none at
+ # all, but a tag reached from the front can only be showing its header,
+ # since a footer sits at the very end. A tag with neither begins with
+ # item data, does not match here, and the file is decoded instead.
+ return int.from_bytes(tail[12:16], "little") + 32
+ return 0
+
+
+def _mp3_seconds(raw: bytes, cap: float) -> Optional[float]:
+ """Seconds of audio an MPEG stream holds, walking frame headers only.
+
+ Returns as soon as the running total passes `cap`, so proving a file too long
+ costs a fraction of the walk.
+
+ None means "this walk established no length", and the caller has to bound the
+ file some other way rather than forward it. That covers a stream with no
+ readable frame, a free-format frame (bitrate index 0 carries no length), and
+ a walk that hit bytes it could not read partway through. The last one is what
+ matters: a decoder resynchronises past junk and plays the rest, so returning
+ only what was counted before it let four stray bytes present half an hour of
+ audio as two seconds. A known metadata trailer is not junk and still ends the
+ count cleanly.
+ """
+ offset = _id3_tag_length(raw)
+ seconds = 0.0
+ frames = 0
+ while offset + 4 <= len(raw):
+ first, second, third = raw[offset], raw[offset + 1], raw[offset + 2]
+ if first != 0xFF or (second & 0xE0) != 0xE0:
+ break
+ version = (second >> 3) & 0x03
+ layer = (second >> 1) & 0x03
+ if version == 0x01 or layer != 0x01:
+ break
+ bitrate = _MPEG_LAYER3_BITRATES[version][(third >> 4) & 0x0F] * 1000
+ rate = _MPEG_SAMPLE_RATES[version][(third >> 2) & 0x03]
+ if bitrate <= 0 or rate <= 0:
+ break
+ # MPEG-1 Layer III carries 1152 samples per frame, MPEG-2 and 2.5 half that.
+ samples = 1152 if version == 0x03 else 576
+ length = (samples // 8) * bitrate // rate + ((third >> 1) & 0x01)
+ if length <= 4:
+ break
+ seconds += samples / rate
+ frames += 1
+ if seconds > cap:
+ return seconds
+ offset += length
+ if not frames:
+ return None
+ # Fewer than four bytes cannot begin a frame, so a short tail is the end.
+ # Anything longer has to account for itself all the way to EOF: a trailer
+ # that stops short is followed by something this walk never read.
+ while offset < len(raw):
+ if len(raw) - offset < 4:
+ return seconds
+ length = _mp3_trailer_length(raw, offset)
+ if length <= 0:
+ return None
+ offset += length
+ return seconds
+
+
+def _passthrough_audio_seconds(raw: bytes, container: str, cap: float) -> Optional[float]:
+ """How long a container forwarded untouched runs, or None if unreadable."""
+ if container == "wav":
+ return _wav_seconds(raw)
+ if container == "mp3":
+ return _mp3_seconds(raw, cap)
return None
@@ -17754,20 +18089,27 @@ def _mono_f32_to_wav_bytes(arr: "np.ndarray", sample_rate: int) -> bytes:
import numpy as np
import wave
- arr = np.nan_to_num(np.asarray(arr, dtype = np.float32).flatten(), posinf = 0.0, neginf = 0.0)
- if arr.size == 0:
+ if np.asarray(arr).size == 0:
raise ValueError("decoded audio is empty")
- peak = float(np.abs(arr).max())
+ # One copy, then every stage in place. Flatten, the nan scrub, abs, the
+ # normalise and the scale each used to allocate the whole recording again.
+ scratch = np.nan_to_num(np.asarray(arr, dtype = np.float32).ravel(), posinf = 0.0, neginf = 0.0)
+ # Every value is finite after the scrub, so the extremes carry the peak
+ # without an absolute copy of the array.
+ peak = max(abs(float(scratch.min())), abs(float(scratch.max())))
if peak > 1.0:
- arr = arr / peak
- pcm = (arr * 32767.0).astype(np.int16)
+ np.divide(scratch, peak, out = scratch)
+ np.multiply(scratch, 32767.0, out = scratch)
+ pcm = scratch.astype(np.int16)
+ del scratch
buf = io.BytesIO()
with wave.open(buf, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(int(sample_rate))
- wf.writeframes(pcm.tobytes())
+ # The array itself, not a bytes copy of it: wave casts the buffer.
+ wf.writeframes(pcm)
return buf.getvalue()
@@ -17777,13 +18119,37 @@ def _resample_mono_linear(arr: "np.ndarray", source_rate: int, target_rate: int)
if source_rate <= 0 or target_rate <= 0 or source_rate == target_rate:
return arr
- duration = len(arr) / float(source_rate)
+ count = len(arr)
+ if count == 0:
+ return arr
+ duration = count / float(source_rate)
target_len = max(1, int(round(duration * target_rate)))
- if target_len == len(arr):
+ if target_len == count:
return arr
- source_x = np.linspace(0.0, duration, num = len(arr), endpoint = False)
- target_x = np.linspace(0.0, duration, num = target_len, endpoint = False)
- return np.interp(target_x, source_x, arr).astype(np.float32)
+ # Both grids are uniform and start at zero, so a slice of either is
+ # `index * step` and can be built where it is needed. Materializing them
+ # whole cost two float64 arrays the length of the recording, 8 bytes per
+ # input sample each, on top of np.interp's own float64 result: the single
+ # largest transient in the transcode, and larger than the audio itself.
+ source_step = duration / count
+ target_step = duration / target_len
+ out = np.empty(target_len, dtype = np.float32)
+ # Sized from the source window a slice reads, not from the slice: at 3:1 a
+ # million outputs consult three million inputs, and it is the window that
+ # np.interp widens to float64.
+ ratio = max(1.0, count / target_len)
+ slice_len = max(1, int(_RESAMPLE_SLICE_SAMPLES / ratio))
+ for start in range(0, target_len, slice_len):
+ stop = min(start + slice_len, target_len)
+ target_x = np.arange(start, stop, dtype = np.float64) * target_step
+ # The window np.interp consults for this slice, widened by a sample at
+ # each end so the bracketing points, and the clamp past the last one,
+ # are the ones the whole-array pass would have found.
+ first = max(0, int(target_x[0] / source_step) - 1)
+ last = min(count, int(target_x[-1] / source_step) + 3)
+ source_x = np.arange(first, last, dtype = np.float64) * source_step
+ out[start:stop] = np.interp(target_x, source_x, arr[first:last])
+ return out
def _fit_transcoded_audio_to_wav_cap(
@@ -17808,47 +18174,246 @@ def _fit_transcoded_audio_to_wav_cap(
return fitted, target_rate
+def _decode_audio_mono_with_soundfile(raw: bytes) -> "tuple[np.ndarray, int]":
+ """Decode libsndfile-supported audio in bounded blocks."""
+ import io
+
+ import numpy as np
+ import soundfile as sf
+
+ sample_count = 0
+ joined = None
+ with sf.SoundFile(io.BytesIO(raw)) as source:
+ sample_rate = int(source.samplerate)
+ if sample_rate <= 0:
+ raise ValueError("decoded audio has an invalid sample rate")
+ # blocks() sizes its buffer as frames * channels and yields a copy of
+ # it, so a frame budget alone lets a many-channel container hold far
+ # more than the mono case it was chosen for: 48k frames of 255-channel
+ # Vorbis is 49 MB twice over, from an upload capped at 25. The samples
+ # kept are mono, since each block is downmixed below; this bounds the
+ # transient. Ordinary audio is unaffected, the divisor only biting past
+ # about 20 channels.
+ channels = max(1, int(getattr(source, "channels", 1) or 1))
+ block_frames = max(1, min(sample_rate, 65_536, _MAX_DECODE_BLOCK_SAMPLES // channels))
+ ceiling = _decoded_sample_ceiling(sample_rate)
+ # The length the header claims, kept only while it is one this decode is
+ # allowed to reach. It is a hint that saves growth copies, never a
+ # promise: a header that under-counts, over-counts or says nothing at all
+ # only changes how often the buffer is resized.
+ declared = int(getattr(source, "frames", 0) or 0)
+ if declared > ceiling:
+ declared = 0
+ for block in source.blocks(
+ blocksize = block_frames,
+ dtype = "float32",
+ always_2d = False,
+ ):
+ sample_count += len(block)
+ if sample_count > ceiling:
+ raise _DecodedAudioTooLongError(
+ f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+ if block.ndim > 1:
+ block = block.mean(axis = 1)
+ if joined is None:
+ joined = np.empty(min(max(len(block), 1), ceiling), dtype = np.float32)
+ if sample_count > len(joined):
+ # Doubling keeps the resizes amortised when the header is absent
+ # or short. Adopting the declared length instead, once it is the
+ # larger target, spends one allocation on an honest header rather
+ # than a run of them; not before the decode has produced half of
+ # it, because sizing a buffer from an unread header is how a small
+ # file asks for a large allocation.
+ target = max(sample_count, 2 * len(joined))
+ if declared > target and sample_count * 2 >= declared:
+ target = declared
+ grown = np.empty(min(target, ceiling), dtype = np.float32)
+ grown[: sample_count - len(block)] = joined[: sample_count - len(block)]
+ joined = grown
+ # One append-only buffer, so the samples are in decode order by
+ # construction. The previous shape kept overflow blocks in a separate
+ # list and re-tested the fit per block, which let a short final block
+ # drop back into the space left over and be emitted ahead of the
+ # blocks that overflowed before it: the audio was reordered while its
+ # length stayed right, so nothing downstream could notice.
+ joined[sample_count - len(block) : sample_count] = block
+ if not sample_count:
+ raise ValueError("audio container decoded to no samples")
+ # A slice keeps the whole allocation alive, which is free when the length was
+ # known and wasteful when the buffer had to be guessed at.
+ if sample_count * 2 < len(joined):
+ return joined[:sample_count].copy(), sample_rate
+ return joined[:sample_count], sample_rate
+
+
+def _decoded_sample_ceiling(sample_rate: int) -> int:
+ """The most samples a decode may produce, in one place.
+
+ Two limits apply at once and neither implies the other: the clock, and a
+ rate-independent sample count that keeps a high-rate file from spending
+ thirty minutes' worth of memory. Nothing may allocate past the smaller of
+ them, because that is the point at which the decode refuses anyway.
+ """
+ return min(sample_rate * _MAX_AUDIO_SECONDS, _MAX_DECODED_SAMPLES)
+
+
+def _av_expected_samples(container, sample_rate: int, ceiling: int) -> int:
+ """Samples to size the output buffer for, from the container's own duration.
+
+ The duration is a hint, not a promise: it can be absent, wrong, or a lie. It
+ only decides the first allocation, so being wrong costs a growth copy or some
+ untouched address space, never a wrong result. It is still clamped to the
+ ceiling, because a declared duration the decode would refuse must not be
+ allowed to size an allocation on the way to refusing it.
+ """
+ import av
+
+ micros = getattr(container, "duration", None) or 0
+ seconds = float(micros) / float(av.time_base) if micros > 0 else 0.0
+ if seconds <= 0.0:
+ stream = container.streams.audio[0]
+ if stream.duration and stream.time_base:
+ seconds = float(stream.duration * stream.time_base)
+ if not (seconds > 0.0):
+ # Nothing declared, so start at a minute and grow.
+ seconds = 60.0
+ return min(int(seconds * sample_rate) + 1, ceiling)
+
+
+def _decode_audio_mono_with_av(raw: bytes) -> "tuple[np.ndarray, int]":
+ """Decode an audio container with PyAV's bundled FFmpeg libraries."""
+ import io
+
+ import av
+ import numpy as np
+
+ # Collecting every decoded block and joining at the end costs two copies of
+ # the recording at once, whether the join is np.concatenate or a fill loop
+ # that drops blocks as it copies: the blocks are small enough that freeing
+ # them returns them to the allocator's free lists rather than to the OS, so
+ # resident memory only ever grows. Measured at 2.0x either way for a
+ # 30-minute 48 kHz upload. Writing each block into one output buffer as it
+ # arrives is the only shape that keeps a single copy live.
+ joined = None
+ sample_rate = 0
+ sample_count = 0
+ ceiling = 0
+ resampler = None
+
+ def append_resampled(resampled) -> None:
+ nonlocal joined, sample_count
+ block = resampled.to_ndarray().reshape(-1)
+ sample_count += len(block)
+ if sample_count > ceiling:
+ raise _DecodedAudioTooLongError(
+ f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+ if sample_count > len(joined):
+ # The container under-reported its duration, or never declared one.
+ # Doubling makes the growth copies amortised, and the transient is
+ # the old buffer plus the new one rather than the whole decode. The
+ # doubling is clamped too: the count that survived the check above is
+ # within the ceiling, but twice the buffer holding it need not be.
+ grown = np.empty(min(max(sample_count, 2 * len(joined)), ceiling), dtype = np.float32)
+ grown[: sample_count - len(block)] = joined[: sample_count - len(block)]
+ joined = grown
+ joined[sample_count - len(block) : sample_count] = block
+
+ with av.open(io.BytesIO(raw), mode = "r", metadata_errors = "ignore") as container:
+ if not container.streams.audio:
+ raise ValueError("audio container has no audio stream")
+ for frame in container.decode(audio = 0):
+ if resampler is None:
+ sample_rate = int(frame.sample_rate or 0)
+ if sample_rate <= 0:
+ raise ValueError("decoded audio has an invalid sample rate")
+ resampler = av.AudioResampler(
+ format = "flt",
+ layout = "mono",
+ rate = sample_rate,
+ )
+ ceiling = _decoded_sample_ceiling(sample_rate)
+ joined = np.empty(
+ _av_expected_samples(container, sample_rate, ceiling), dtype = np.float32
+ )
+ for resampled in resampler.resample(frame):
+ append_resampled(resampled)
+ if resampler is not None:
+ for resampled in resampler.resample(None):
+ append_resampled(resampled)
+ if not sample_count:
+ raise ValueError("audio container decoded to no samples")
+ # A slice keeps the whole allocation alive, which is free when the declared
+ # duration was honest and wasteful when it was not.
+ if sample_count * 2 < len(joined):
+ return joined[:sample_count].copy(), sample_rate
+ return joined[:sample_count], sample_rate
+
+
def _decode_audio_mono(raw: bytes) -> "tuple[np.ndarray, int]":
"""Decode audio bytes to (mono float32 array, native sample_rate).
- soundfile (libsndfile) reads wav/mp3/ogg/flac straight from memory. librosa
- (ffmpeg-backed) additionally covers m4a/webm but needs a real path and is
- absent on no-torch GGUF-only installs. Both imports are inside the fallback
- so a missing decoder degrades to the next one (and finally a clear error)
- rather than crashing.
+ soundfile (libsndfile) reads wav/mp3/ogg/flac in bounded blocks. PyAV is
+ installed in every Studio environment and uses its bundled FFmpeg libraries
+ for containers including m4a/webm/WMA/AMR. librosa remains the final fallback
+ in full ML environments, but is absent from no-torch GGUF-only installs.
"""
- import io
-
try:
- import soundfile as sf
- arr, sr = sf.read(io.BytesIO(raw), dtype = "float32")
+ arr, sr = _decode_audio_mono_with_soundfile(raw)
+ except _DecodedAudioTooLongError:
+ raise
except Exception:
try:
- import librosa
- except ModuleNotFoundError as e:
- raise RuntimeError(
- "this audio format needs librosa, which is not installed in "
- "GGUF-only environments; use wav, mp3, ogg or flac"
- ) from e
- import os
- import tempfile
- from utils.paths import ensure_dir, tmp_root
-
- with tempfile.NamedTemporaryFile(
- suffix = ".audio",
- delete = False,
- dir = str(ensure_dir(tmp_root())),
- ) as tmp:
- tmp.write(raw)
- tmp_path = tmp.name
- try:
- arr, sr = librosa.load(tmp_path, sr = None, mono = True)
- finally:
- os.unlink(tmp_path)
+ arr, sr = _decode_audio_mono_with_av(raw)
+ except _DecodedAudioTooLongError:
+ raise
+ except Exception as av_error:
+ try:
+ import librosa
+ except ModuleNotFoundError:
+ raise RuntimeError(
+ "this audio format could not be decoded; run `unsloth studio update` "
+ "to install PyAV or convert it to wav, mp3, ogg or flac"
+ ) from av_error
+ import os
+ import tempfile
+ from utils.paths import ensure_dir, tmp_root
+
+ with tempfile.NamedTemporaryFile(
+ suffix = ".audio",
+ delete = False,
+ dir = str(ensure_dir(tmp_root())),
+ ) as tmp:
+ tmp.write(raw)
+ tmp_path = tmp.name
+ try:
+ # audioread/FFmpeg would otherwise materialize the whole
+ # waveform before the check below. Ask for the native rate first
+ # so the window covers the sample ceiling as well as the clock.
+ try:
+ probe_rate = int(librosa.get_samplerate(tmp_path) or 0)
+ except Exception: # noqa: BLE001 - handled below
+ probe_rate = 0
+ if probe_rate <= 0:
+ # Without the rate there is no window that bounds the read:
+ # 30 minutes is gigabytes at a high rate and nothing at a low
+ # one. This is the last fallback, after soundfile and PyAV
+ # both declined, so refusing costs a file nothing else reads.
+ raise RuntimeError(
+ "this audio file does not report a sample rate, so it cannot "
+ "be decoded within the size limit; convert it to wav or mp3"
+ )
+ window = min(float(_MAX_AUDIO_SECONDS + 1), _MAX_DECODED_SAMPLES / probe_rate + 1)
+ arr, sr = librosa.load(tmp_path, sr = None, mono = True, duration = window)
+ finally:
+ os.unlink(tmp_path)
if arr.ndim > 1:
arr = arr.mean(axis = 1)
- if sr > 0 and len(arr) > sr * _MAX_AUDIO_SECONDS:
- raise ValueError(f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit")
+ if (sr > 0 and len(arr) > sr * _MAX_AUDIO_SECONDS) or len(arr) > _MAX_DECODED_SAMPLES:
+ raise _DecodedAudioTooLongError(
+ f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
return arr, sr
@@ -17865,7 +18430,24 @@ def _prepare_audio_for_llama(b64: str) -> tuple[str, str]:
raw = base64.b64decode(b64)
passthrough = _sniff_audio_container(raw)
if passthrough is not None:
- return b64, passthrough
+ # Forwarding skips every bounded decoder, so the duration cap has to be
+ # applied from the headers instead. A 16 kbps MP3 holds hours inside the
+ # 25 MB upload cap, and llama-server was left to decode all of it. A
+ # 25 MB upload cap, and llama-server was left to decode all of it.
+ seconds = _passthrough_audio_seconds(raw, passthrough, _MAX_AUDIO_SECONDS)
+ if seconds is not None and seconds > _MAX_AUDIO_SECONDS:
+ raise _DecodedAudioTooLongError(
+ f"audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+ # Headers that cannot state a length do not earn a free pass. Forwarding
+ # them anyway meant the cap held only for containers honest enough to
+ # describe themselves, which is the wrong way round: four junk bytes in
+ # an MPEG stream, or a WAV with no data chunk, ended the header walk and
+ # took the whole recording through with it. Decoding costs a transcode
+ # and nothing else, and puts the file back under both ceilings, so it
+ # still reaches the model.
+ if seconds is not None:
+ return b64, passthrough
arr, sr = _decode_audio_mono(raw)
arr, sr = _fit_transcoded_audio_to_wav_cap(arr, sr)
@@ -20117,6 +20699,13 @@ async def produce_openai_chat_completions(
)
system_prompt, chat_messages, _ = _extract_content_parts(payload.messages)
system_prompt = _apply_current_date_prompt(system_prompt, request)
+ except _DecodedAudioTooLongError as e:
+ # A limit the caller can act on, not a server fault.
+ api_monitor.fail(monitor_id, str(e))
+ raise HTTPException(
+ status_code = 413,
+ detail = _audio_too_long_detail(),
+ )
except Exception as e:
api_monitor.fail(monitor_id, _friendly_error(e))
raise
@@ -20561,6 +21150,11 @@ async def produce_openai_chat_completions(
if _preprepared_audio is not None
else await asyncio.to_thread(_prepare_audio_for_llama, payload.audio_base64)
)
+ except _DecodedAudioTooLongError as e:
+ # A valid file that is simply too long reports the limit, as the
+ # non-GGUF path does, rather than reading as corrupt audio.
+ logger.info("Audio rejected at the duration limit: %s", e)
+ raise _reject(413, _audio_too_long_detail())
except Exception as e:
logger.warning("Audio decode failed: %s", e, exc_info = True)
raise _reject(400, "Could not decode the provided audio file.")
diff --git a/studio/backend/tests/test_llama_audio_attachment_container.py b/studio/backend/tests/test_llama_audio_attachment_container.py
new file mode 100644
index 00000000000..a6ab9e3ce34
--- /dev/null
+++ b/studio/backend/tests/test_llama_audio_attachment_container.py
@@ -0,0 +1,1377 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import base64
+import io
+import pathlib
+import sys
+import types
+
+import numpy as np
+import pytest
+import routes.inference as inference_route
+
+
+def _id3_prefix() -> bytes:
+ # ID3v2.4, no payload and no footer.
+ return b"ID3\x04\x00\x00\x00\x00\x00\x00"
+
+
+def test_audio_container_sniffer_distinguishes_mpeg_layers_and_adts():
+ # MPEG-1 Layer III, MPEG-1 Layer II, and ADTS AAC respectively.
+ assert inference_route._sniff_audio_container(b"\xff\xfb\x90\x64") == "mp3"
+ assert inference_route._sniff_audio_container(b"\xff\xfd\x90\x64") is None
+ assert inference_route._sniff_audio_container(b"\xff\xf1\x50\x80") is None
+
+
+def test_audio_container_sniffer_checks_the_frame_after_id3():
+ assert inference_route._sniff_audio_container(_id3_prefix() + b"\xff\xfb\x90\x64") == "mp3"
+ assert inference_route._sniff_audio_container(_id3_prefix() + b"\xff\xfd\x90\x64") is None
+ assert inference_route._sniff_audio_container(_id3_prefix() + b"\xff\xf1\x50\x80") is None
+
+
+def test_aac_and_mp2_are_transcoded_instead_of_forwarded_as_mp3(monkeypatch):
+ decoded = []
+ monkeypatch.setattr(
+ inference_route,
+ "_decode_audio_mono",
+ lambda raw: (decoded.append(raw) or object(), 16_000),
+ )
+ monkeypatch.setattr(
+ inference_route,
+ "_fit_transcoded_audio_to_wav_cap",
+ lambda array, sample_rate: (array, sample_rate),
+ )
+ monkeypatch.setattr(
+ inference_route,
+ "_mono_f32_to_wav_bytes",
+ lambda _array, _sample_rate: b"transcoded wav",
+ )
+
+ for raw in (b"\xff\xfd\x90\x64", b"\xff\xf1\x50\x80"):
+ encoded, format_name = inference_route._prepare_audio_for_llama(
+ base64.b64encode(raw).decode("ascii")
+ )
+ assert format_name == "wav"
+ assert base64.b64decode(encoded) == b"transcoded wav"
+
+ assert decoded == [b"\xff\xfd\x90\x64", b"\xff\xf1\x50\x80"]
+
+
+def _encode_wma() -> bytes:
+ # PyAV is optional in a backend install, so only the fixtures that need it skip.
+ av = pytest.importorskip("av")
+ buffer = io.BytesIO()
+ with av.open(buffer, mode = "w", format = "asf") as output:
+ stream = output.add_stream("wmav2", rate = 44_100)
+ stream.layout = "stereo"
+ stream.bit_rate = 128_000
+ indexes = np.arange(8_192)
+ mono = np.sin(indexes * 2 * np.pi * 440 / 44_100).astype(np.float32)
+ frame = av.AudioFrame.from_ndarray(
+ np.stack([mono, mono]),
+ format = "fltp",
+ layout = "stereo",
+ )
+ frame.sample_rate = 44_100
+ for packet in stream.encode(frame):
+ output.mux(packet)
+ for packet in stream.encode():
+ output.mux(packet)
+ return buffer.getvalue()
+
+
+def _encode_amr() -> bytes:
+ av = pytest.importorskip("av")
+ buffer = io.BytesIO()
+ with av.open(buffer, mode = "w", format = "amr") as output:
+ stream = output.add_stream("amr_nb", rate = 8_000)
+ stream.layout = "mono"
+ stream.bit_rate = 12_200
+ indexes = np.arange(3_200)
+ samples = (np.sin(indexes * 2 * np.pi * 440 / 8_000) * 16_000).astype(np.int16)
+ frame = av.AudioFrame.from_ndarray(
+ samples.reshape(1, -1),
+ format = "s16",
+ layout = "mono",
+ )
+ frame.sample_rate = 8_000
+ for packet in stream.encode(frame):
+ output.mux(packet)
+ for packet in stream.encode():
+ output.mux(packet)
+ return buffer.getvalue()
+
+
+def test_wma_and_amr_decode_with_no_librosa(monkeypatch):
+ # Match the GGUF-only installation: PyAV is present, librosa is not. Force
+ # libsndfile out too so the test proves the PyAV fallback owns both formats.
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", None)
+
+ for raw, expected_rate in ((_encode_wma(), 44_100), (_encode_amr(), 8_000)):
+ samples, sample_rate = inference_route._decode_audio_mono(raw)
+ assert sample_rate == expected_rate
+ assert samples.ndim == 1
+ assert samples.dtype == np.float32
+ assert np.max(np.abs(samples)) > 0
+
+
+def test_pyav_output_is_not_held_twice(monkeypatch):
+ """np.concatenate keeps the block list and its result live at once, so a
+ 30-minute upload at the sample ceiling held two 346 MB arrays. Each block is
+ written into the output as it arrives, in decode order, and nothing joins a
+ list at the end."""
+ raw = _encode_amr()
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", None)
+
+ def refuse(*_args, **_kwargs):
+ raise AssertionError("the PyAV path must not concatenate its blocks")
+
+ monkeypatch.setattr(np, "concatenate", refuse)
+ samples, sample_rate = inference_route._decode_audio_mono(raw)
+ assert sample_rate == 8_000
+ assert samples.dtype == np.float32
+ assert np.max(np.abs(samples)) > 0
+
+
+def test_pyav_duration_limit_stops_before_concatenating(monkeypatch):
+ raw = _encode_amr()
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", None)
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 0)
+
+ def concatenate_after_limit(*_args, **_kwargs):
+ raise AssertionError("duration-limited audio must not be concatenated")
+
+ monkeypatch.setattr(np, "concatenate", concatenate_after_limit)
+ try:
+ inference_route._decode_audio_mono(raw)
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected PyAV to stop at the decoded-duration limit")
+
+
+def test_soundfile_duration_limit_stops_before_concatenating(monkeypatch):
+ decoded_blocks = []
+
+ class FakeSoundFile:
+ samplerate = 4
+
+ def __init__(self, _source):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def blocks(self, **_kwargs):
+ for block in (np.ones((4, 2), dtype = np.float32), np.ones((1, 2), dtype = np.float32)):
+ decoded_blocks.append(len(block))
+ yield block
+
+ monkeypatch.setitem(
+ sys.modules,
+ "soundfile",
+ types.SimpleNamespace(SoundFile = FakeSoundFile),
+ )
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)
+
+ def concatenate_after_limit(*_args, **_kwargs):
+ raise AssertionError("duration-limited audio must not be concatenated")
+
+ monkeypatch.setattr(np, "concatenate", concatenate_after_limit)
+ try:
+ inference_route._decode_audio_mono(b"bounded soundfile input")
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected SoundFile to stop at the decoded-duration limit")
+
+ assert decoded_blocks == [4, 1]
+
+
+def test_audio_input_decode_is_bounded_by_the_duration_limit(monkeypatch):
+ """The non-GGUF audio-input path decodes with torchaudio, which used to
+ materialize the whole waveform before any duration check. A low-bitrate AMR
+ or WMA under the 25 MB upload cap can still hold hours of PCM."""
+ loads: list[dict] = []
+
+ class _FakeTorchaudio:
+ @staticmethod
+ def info(_path):
+ return types.SimpleNamespace(sample_rate = 16_000, num_frames = 16_000 * 60 * 60)
+
+ @staticmethod
+ def load(path, **kwargs):
+ loads.append(kwargs)
+ raise AssertionError("an over-limit file must not be decoded")
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 30 * 60)
+
+ try:
+ inference_route._decode_audio_base64(base64.b64encode(b"long recording").decode())
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected the decoded-duration limit to be enforced")
+ assert loads == []
+
+
+def test_audio_input_decode_streams_a_container_that_hides_its_length(monkeypatch):
+ """A header that does not report num_frames cannot be bounded by num_frames:
+ with several channels the read would still outrun the ceiling, and trimming it
+ would silently truncate a file that is inside the limit. Stream it instead."""
+ streamed: list[bytes] = []
+
+ class _FakeTorchaudio:
+ @staticmethod
+ def info(_path):
+ return types.SimpleNamespace(sample_rate = 16_000, num_frames = 0, num_channels = 2)
+
+ @staticmethod
+ def load(*_a, **_k):
+ raise AssertionError("an unreported length must not be loaded whole")
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+
+ def _bounded(raw):
+ streamed.append(raw)
+ return np.zeros(16_000, dtype = np.float32), 16_000
+
+ monkeypatch.setattr(inference_route, "_decode_audio_mono", _bounded)
+
+ out = inference_route._decode_audio_base64(base64.b64encode(b"endless stream").decode())
+ assert out.shape == (16_000,)
+ assert streamed == [b"endless stream"]
+
+
+def test_a_wide_container_inside_the_clock_is_streamed_not_refused(monkeypatch):
+ """30 minutes of high-rate or multichannel audio is within the advertised
+ limit, so it must still decode; it just cannot be held at once."""
+ streamed: list[bytes] = []
+
+ class _FakeTorchaudio:
+ @staticmethod
+ def info(_path):
+ # Inside the 30-minute clock, past the ceiling once channels count.
+ return types.SimpleNamespace(
+ sample_rate = 48_000,
+ num_frames = 48_000 * 20 * 60,
+ num_channels = 2,
+ )
+
+ @staticmethod
+ def load(*_a, **_k):
+ raise AssertionError("a wide container must not be materialized")
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+
+ def _bounded(raw):
+ streamed.append(raw)
+ return np.zeros(16_000, dtype = np.float32), 16_000
+
+ monkeypatch.setattr(inference_route, "_decode_audio_mono", _bounded)
+
+ out = inference_route._decode_audio_base64(base64.b64encode(b"wide input").decode())
+ assert out.shape == (16_000,)
+ assert streamed == [b"wide input"]
+
+
+def test_audio_input_decode_passes_a_short_recording_through(monkeypatch):
+ # torch is only a source of fake tensors for the fake torchaudio below, so
+ # a machine without it should skip rather than fail, as the av cases do.
+ torch = pytest.importorskip("torch")
+
+ class _FakeTorchaudio:
+ transforms = types.SimpleNamespace()
+
+ @staticmethod
+ def info(_path):
+ return types.SimpleNamespace(sample_rate = 16_000, num_frames = 16_000)
+
+ @staticmethod
+ def load(_path, **_kwargs):
+ return torch.ones((2, 16_000)), 16_000
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+ out = inference_route._decode_audio_base64(base64.b64encode(b"short").decode())
+ assert out.shape == (16_000,)
+ assert out.dtype == np.float32
+
+
+def test_a_load_is_capped_by_the_sample_ceiling_not_only_by_the_clock(monkeypatch):
+ """num_frames is the value being distrusted, so a container that understates
+ it must not license a read up to the rate-relative limit: at 192 kHz that is
+ four times the sample ceiling."""
+ # torch is only a source of fake tensors for the fake torchaudio below, so
+ # a machine without it should skip rather than fail, as the av cases do.
+ torch = pytest.importorskip("torch")
+
+ loads: list[dict] = []
+ ceiling = 64
+
+ class _FakeTorchaudio:
+ transforms = types.SimpleNamespace()
+
+ @staticmethod
+ def info(_path):
+ # Claims 8 frames; the file actually holds far more.
+ return types.SimpleNamespace(sample_rate = 192_000, num_frames = 8, num_channels = 2)
+
+ @staticmethod
+ def load(_path, **kwargs):
+ loads.append(kwargs)
+ frames = kwargs["num_frames"]
+ return torch.ones((2, frames)), 192_000
+
+ monkeypatch.setattr(inference_route, "_MAX_DECODED_SAMPLES", ceiling)
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+
+ try:
+ inference_route._decode_audio_base64(base64.b64encode(b"understated").decode())
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected the sample ceiling to be enforced")
+ # One frame past ceiling / channels, not past rate * seconds.
+ assert loads == [{"num_frames": ceiling // 2 + 1}]
+
+
+def test_an_honest_length_is_still_read_in_full(monkeypatch):
+ """Capping by both limits must not truncate a file that fits: the read window
+ stays at or above what info() reported."""
+ # torch is only a source of fake tensors for the fake torchaudio below, so
+ # a machine without it should skip rather than fail, as the av cases do.
+ torch = pytest.importorskip("torch")
+
+ loads: list[dict] = []
+
+ class _FakeTorchaudio:
+ transforms = types.SimpleNamespace()
+
+ @staticmethod
+ def info(_path):
+ return types.SimpleNamespace(sample_rate = 48_000, num_frames = 48_000 * 60, num_channels = 1)
+
+ @staticmethod
+ def load(_path, **kwargs):
+ loads.append(kwargs)
+ return torch.ones((1, 48_000 * 60)), 48_000
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+
+ def _must_not_stream(_raw):
+ raise AssertionError("a file that fits must not be streamed")
+
+ monkeypatch.setattr(inference_route, "_decode_audio_mono", _must_not_stream)
+
+ class _FakeResample:
+ def __init__(self, **_kwargs):
+ pass
+
+ def __call__(self, waveform):
+ return waveform[..., :16_000]
+
+ _FakeTorchaudio.transforms.Resample = _FakeResample
+ out = inference_route._decode_audio_base64(base64.b64encode(b"one minute").decode())
+ assert out.shape == (16_000,)
+ assert loads[0]["num_frames"] >= 48_000 * 60
+
+
+def test_audio_input_decode_stays_bounded_when_the_probe_fails(monkeypatch):
+ """info() failing does not license an unbounded load: a container it cannot
+ read is still handed to the bounded decoder rather than materialized whole."""
+ bounded: list[bytes] = []
+
+ class _FakeTorchaudio:
+ transforms = types.SimpleNamespace()
+
+ @staticmethod
+ def info(_path):
+ raise RuntimeError("unknown container")
+
+ @staticmethod
+ def load(*_a, **_k):
+ raise AssertionError("a probe failure must not fall back to a whole load")
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+
+ def _bounded(raw):
+ bounded.append(raw)
+ return np.zeros(16_000, dtype = np.float32), 16_000
+
+ monkeypatch.setattr(inference_route, "_decode_audio_mono", _bounded)
+
+ out = inference_route._decode_audio_base64(base64.b64encode(b"amr bytes").decode())
+ assert out.shape == (16_000,)
+ assert bounded == [b"amr bytes"]
+
+
+def test_the_probe_failure_path_still_enforces_the_cap(monkeypatch):
+ """The bounded decoder raises at the cap, and that error is not swallowed."""
+
+ class _FakeTorchaudio:
+ @staticmethod
+ def info(_path):
+ raise RuntimeError("unknown container")
+
+ @staticmethod
+ def load(*_a, **_k):
+ raise AssertionError("a probe failure must not fall back to a whole load")
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+
+ def _too_long(_raw):
+ raise inference_route._DecodedAudioTooLongError("decoded audio exceeds the limit")
+
+ monkeypatch.setattr(inference_route, "_decode_audio_mono", _too_long)
+
+ try:
+ inference_route._decode_audio_base64(base64.b64encode(b"hours of amr").decode())
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected the duration cap to be enforced")
+
+
+def test_a_high_rate_container_is_capped_by_samples_not_only_by_clock(monkeypatch):
+ """The duration cap is rate-relative, so 30 minutes at a high rate retains far
+ more memory than the same speech at 16 kHz. The absolute ceiling bounds it."""
+ decoded_blocks = []
+
+ class FakeSoundFile:
+ samplerate = 192_000
+ channels = 1
+
+ def __init__(self, *_a, **_k):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_a):
+ return False
+
+ def blocks(self, **_kwargs):
+ # Well inside the 30-minute clock at this rate, past the sample cap.
+ block = np.ones(ceiling + 1, dtype = np.float32)
+ decoded_blocks.append(len(block))
+ yield block
+
+ # A small ceiling: the check is what is under test, and the production
+ # constant would allocate 346 MB on every run of this file.
+ ceiling = 64
+ monkeypatch.setattr(inference_route, "_MAX_DECODED_SAMPLES", ceiling)
+ monkeypatch.setitem(sys.modules, "soundfile", types.SimpleNamespace(SoundFile = FakeSoundFile))
+
+ def concatenate_after_limit(*_args, **_kwargs):
+ raise AssertionError("an over-ceiling decode must not be concatenated")
+
+ monkeypatch.setattr(np, "concatenate", concatenate_after_limit)
+ try:
+ inference_route._decode_audio_mono_with_soundfile(b"high rate input")
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected the decoded-sample ceiling to be enforced")
+ assert decoded_blocks == [ceiling + 1]
+
+
+def test_the_ceiling_never_bites_before_the_advertised_duration(monkeypatch):
+ """An ordinary rate keeps the full 30 minutes: the ceiling only refuses rates
+ above 48 kHz, so the documented limit still means what it says."""
+ assert inference_route._MAX_DECODED_SAMPLES >= 48_000 * inference_route._MAX_AUDIO_SECONDS
+ for rate in (8_000, 16_000, 44_100, 48_000):
+ assert rate * inference_route._MAX_AUDIO_SECONDS <= inference_route._MAX_DECODED_SAMPLES
+
+
+def test_the_librosa_fallback_reads_only_up_to_the_cap(monkeypatch):
+ """audioread would otherwise materialize the whole waveform and let the check
+ after it run too late."""
+ seen: dict = {}
+
+ class _FakeLibrosa:
+ @staticmethod
+ def get_samplerate(_path):
+ return 192_000
+
+ @staticmethod
+ def load(
+ _path,
+ sr = None,
+ mono = True,
+ duration = None,
+ ):
+ seen["duration"] = duration
+ return np.zeros(16_000, dtype = np.float32), 192_000
+
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", _FakeLibrosa)
+ monkeypatch.setattr(
+ inference_route,
+ "_decode_audio_mono_with_av",
+ lambda raw: (_ for _ in ()).throw(RuntimeError("av cannot read this")),
+ )
+
+ inference_route._decode_audio_mono(b"container only librosa reads")
+ # Bounded by the sample ceiling at this rate, not by the 30-minute clock.
+ assert seen["duration"] is not None
+ assert seen["duration"] <= inference_route._MAX_DECODED_SAMPLES / 192_000 + 1
+ assert seen["duration"] < inference_route._MAX_AUDIO_SECONDS
+
+
+def test_an_unprobeable_rate_is_refused_rather_than_read_unbounded(monkeypatch):
+ """Without a sample rate there is no window that bounds the read: 30 minutes
+ is gigabytes at a high rate. Refusing beats loading it to find out."""
+
+ class _FakeLibrosa:
+ @staticmethod
+ def get_samplerate(_path):
+ raise RuntimeError("cannot probe this container")
+
+ @staticmethod
+ def load(*_a, **_k):
+ raise AssertionError("an unbounded read must not be attempted")
+
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", _FakeLibrosa)
+ monkeypatch.setattr(
+ inference_route,
+ "_decode_audio_mono_with_av",
+ lambda raw: (_ for _ in ()).throw(RuntimeError("av cannot read this")),
+ )
+
+ try:
+ inference_route._decode_audio_mono(b"container with no reported rate")
+ except RuntimeError as error:
+ assert "sample rate" in str(error)
+ else:
+ raise AssertionError("expected an unprobeable rate to be refused")
+
+
+def test_the_block_size_counts_channels_not_only_frames(monkeypatch):
+ """blocks() sizes its buffer as frames * channels and yields a copy, so a
+ frame-only budget let a 255-channel container reach 49 MB twice over from an
+ upload capped at 25."""
+ sizes = []
+
+ class FakeSoundFile:
+ samplerate = 48_000
+ channels = 255
+
+ def __init__(self, *_a, **_k):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_a):
+ return False
+
+ def blocks(self, **kwargs):
+ sizes.append(kwargs["blocksize"])
+ yield np.zeros((kwargs["blocksize"], self.channels), dtype = np.float32)
+
+ monkeypatch.setitem(sys.modules, "soundfile", types.SimpleNamespace(SoundFile = FakeSoundFile))
+ samples, rate = inference_route._decode_audio_mono_with_soundfile(b"many channels")
+ assert rate == 48_000
+ assert samples.ndim == 1
+ block = sizes[0]
+ assert block * FakeSoundFile.channels <= inference_route._MAX_DECODE_BLOCK_SAMPLES
+ assert block >= 1
+
+
+def test_ordinary_audio_keeps_its_block_size(monkeypatch):
+ """The divisor only bites past about 20 channels, so nothing common moves."""
+ sizes = []
+
+ class FakeSoundFile:
+ samplerate = 48_000
+ channels = 1
+
+ def __init__(self, *_a, **_k):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_a):
+ return False
+
+ def blocks(self, **kwargs):
+ sizes.append(kwargs["blocksize"])
+ yield np.zeros(kwargs["blocksize"], dtype = np.float32)
+
+ for channels in (1, 2, 6, 8):
+ FakeSoundFile.channels = channels
+ monkeypatch.setitem(
+ sys.modules, "soundfile", types.SimpleNamespace(SoundFile = FakeSoundFile)
+ )
+ inference_route._decode_audio_mono_with_soundfile(b"ordinary audio")
+ assert sizes == [48_000, 48_000, 48_000, 48_000]
+
+
+def _wav_header(
+ sample_rate: int,
+ channels: int,
+ bits: int,
+ data_bytes: int,
+ byte_rate: int = 0,
+ format_tag: int = 0x0001,
+ sub_format: int = 0,
+) -> bytes:
+ block_align = channels * bits // 8
+ byte_rate = byte_rate or sample_rate * block_align
+ # WAVE_FORMAT_EXTENSIBLE needs its 22-byte extension to name a sub-format.
+ extension = b""
+ if sub_format:
+ extension = (
+ (22).to_bytes(2, "little")
+ + bits.to_bytes(2, "little")
+ + (0).to_bytes(4, "little")
+ + sub_format.to_bytes(2, "little")
+ + bytes(14)
+ )
+ fmt_size = 16 + len(extension)
+ return (
+ b"RIFF"
+ + (20 + fmt_size + data_bytes).to_bytes(4, "little")
+ + b"WAVEfmt "
+ + fmt_size.to_bytes(4, "little")
+ + format_tag.to_bytes(2, "little")
+ + channels.to_bytes(2, "little")
+ + sample_rate.to_bytes(4, "little")
+ + byte_rate.to_bytes(4, "little")
+ + block_align.to_bytes(2, "little")
+ + bits.to_bytes(2, "little")
+ + extension
+ + b"data"
+ + data_bytes.to_bytes(4, "little")
+ )
+
+
+def _mp3_frames(seconds: float, bitrate_kbps: int = 8) -> bytes:
+ """MPEG-2.5 Layer III frames at 8 kHz: 576 samples and 72 bytes each at 8 kbps."""
+ version_bits = 0x00 << 3 # MPEG 2.5
+ layer_bits = 0x01 << 1 # Layer III
+ bitrate_index = (0, 8, 16, 24, 32, 40, 48, 56, 64).index(bitrate_kbps)
+ third = (bitrate_index << 4) | (0x02 << 2) # 8 kHz, no padding
+ header = bytes([0xFF, 0xE0 | version_bits | layer_bits | 0x01, third, 0x00])
+ length = (576 // 8) * bitrate_kbps * 1000 // 8_000
+ frame = header + b"\x00" * (length - 4)
+ return frame * max(1, round(seconds / (576 / 8_000)))
+
+
+def test_a_forwarded_wav_is_bounded_by_its_own_header(monkeypatch):
+ """Passthrough returns before every bounded decoder, so the duration cap has
+ to come from the header."""
+ # A small cap, so the fixture can carry the PCM its header declares rather
+ # than allocating the 24 MB that 50 minutes at 8 kHz would really take.
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)
+ long_wav = _wav_header(8_000, 1, 8, 8_000 * 2) + b"\x80" * (8_000 * 2)
+ try:
+ inference_route._prepare_audio_for_llama(base64.b64encode(long_wav).decode())
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected a forwarded wav to be held to the limit")
+
+ # One inside the limit is still forwarded untouched.
+ short_wav = _wav_header(8_000, 1, 8, 4_000) + b"\x80" * 4_000
+ encoded, container = inference_route._prepare_audio_for_llama(
+ base64.b64encode(short_wav).decode()
+ )
+ assert container == "wav"
+ assert base64.b64decode(encoded) == short_wav
+
+
+def test_a_forwarded_mp3_is_bounded_by_its_frame_headers(monkeypatch):
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 60)
+ try:
+ inference_route._prepare_audio_for_llama(base64.b64encode(_mp3_frames(120)).decode())
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected a forwarded mp3 to be held to the limit")
+
+ encoded, container = inference_route._prepare_audio_for_llama(
+ base64.b64encode(_mp3_frames(10)).decode()
+ )
+ assert container == "mp3"
+
+
+def test_the_frame_walk_stops_at_the_cap(monkeypatch):
+ """Proving a file too long must not cost the whole walk."""
+ hours = _mp3_frames(3 * 60 * 60)
+ assert len(hours) > 1_000_000
+ seconds = inference_route._mp3_seconds(hours, 60.0)
+ assert seconds is not None
+ # Just past the cap, not the file's full three hours.
+ assert 60.0 < seconds < 61.0
+
+
+def test_a_container_that_cannot_say_reports_nothing():
+ """None means the walk established no length, not that there is none."""
+ assert inference_route._wav_seconds(b"RIFF\x00\x00\x00\x00WAVE") is None
+ assert inference_route._mp3_seconds(b"not a frame at all", 60.0) is None
+ # A tag with no frames behind it reads as no audio, not as a duration.
+ assert inference_route._mp3_seconds(_id3_prefix(), 60.0) is None
+
+
+def _decode_instead_of_forwarding(monkeypatch):
+ """Replace the decoder with a marker, so a transcode is visible as one."""
+ monkeypatch.setattr(
+ inference_route,
+ "_decode_audio_mono",
+ lambda _raw: (np.zeros(8_000, dtype = np.float32), 8_000),
+ )
+
+
+def test_a_container_that_cannot_say_is_decoded_rather_than_forwarded(monkeypatch):
+ """Forwarding an unreadable header applied the cap only to files honest
+ enough to describe themselves, which is the wrong way round. A transcode
+ costs a decode and puts the file back under both ceilings."""
+ _decode_instead_of_forwarding(monkeypatch)
+ unreadable = _mp3_frames(5) + b"\x01\x02\x03\x04"
+ assert inference_route._mp3_seconds(unreadable, 60.0) is None
+ _encoded, container = inference_route._prepare_audio_for_llama(
+ base64.b64encode(unreadable).decode()
+ )
+ assert container == "wav"
+
+
+def test_junk_between_frames_cannot_shorten_a_forwarded_mp3(monkeypatch):
+ """A decoder resynchronises past junk and plays the rest. Counting only the
+ frames before it let four stray bytes present two hours as two seconds."""
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 60)
+ _decode_instead_of_forwarding(monkeypatch)
+ one_frame = len(_mp3_frames(576 / 8_000))
+ spiked = _mp3_frames(120)
+ spiked = spiked[: one_frame * 10] + b"\x00\x00\x00\x00" + spiked[one_frame * 10 :]
+ assert inference_route._mp3_seconds(spiked, 60.0) is None
+ _encoded, container = inference_route._prepare_audio_for_llama(
+ base64.b64encode(spiked).decode()
+ )
+ assert container == "wav"
+
+
+def test_a_compressed_wav_is_not_measured_with_pcm_arithmetic(monkeypatch):
+ """Only WAVE_FORMAT_PCM fixes nBlockAlign as one sample frame.
+
+ An IMA ADPCM block holds around 505 frames, so rate * blockAlign overstates
+ its byte rate 505-fold and made three hours of audio read as twenty-one
+ seconds, which is short enough to forward. Nothing here can read a codec's
+ own header, so a non-PCM tag reports no duration and is decoded instead.
+ """
+ rate, block_align, samples_per_block = 8_000, 256, 505
+ byte_rate = round(rate / samples_per_block * block_align)
+ payload = b"\x00" * (byte_rate * 120)
+ adpcm = _wav_header(rate, 1, 4, len(payload), byte_rate = byte_rate, format_tag = 0x11) + payload
+ assert inference_route._wav_seconds(adpcm) is None
+
+ _decode_instead_of_forwarding(monkeypatch)
+ _encoded, container = inference_route._prepare_audio_for_llama(base64.b64encode(adpcm).decode())
+ assert container == "wav"
+
+ # Uncompressed tags keep their header arithmetic, extensible included.
+ for tag in (0x0001, 0x0003):
+ pcm = _wav_header(8_000, 1, 8, 8_000, format_tag = tag) + b"\x80" * 8_000
+ assert inference_route._wav_seconds(pcm) == pytest.approx(1.0, rel = 1e-6)
+
+
+def test_an_extensible_wav_is_measured_by_its_sub_format():
+ """WAVE_FORMAT_EXTENSIBLE carries the real tag in its SubFormat GUID, and a
+ file claiming the tag without the extension has not said what it holds."""
+ payload = b"\x80" * 8_000
+ pcm_sub = _wav_header(8_000, 1, 8, len(payload), format_tag = 0xFFFE, sub_format = 0x0001) + payload
+ assert inference_route._wav_seconds(pcm_sub) == pytest.approx(1.0, rel = 1e-6)
+
+ adpcm_sub = (
+ _wav_header(8_000, 1, 8, len(payload), format_tag = 0xFFFE, sub_format = 0x0011) + payload
+ )
+ assert inference_route._wav_seconds(adpcm_sub) is None
+
+ bare = _wav_header(8_000, 1, 8, len(payload), format_tag = 0xFFFE) + payload
+ assert inference_route._wav_seconds(bare) is None
+
+
+def test_the_last_resort_decoder_reads_a_bounded_range(monkeypatch):
+ """torchaudio 2.9's load() is load_with_torchcodec, which calls
+ get_all_samples() and only then slices to num_frames, so the argument bounds
+ the return value and not the allocation. The fallback asks torchcodec for
+ the metadata and reads a range instead."""
+ # torch is only a source of fake tensors for the fake torchaudio below, so
+ # a machine without it should skip rather than fail, as the av cases do.
+ torch = pytest.importorskip("torch")
+
+ ranges: list = []
+
+ class _Metadata:
+ sample_rate = 16_000
+ num_channels = 1
+ duration_seconds = 5.0
+
+ class _Samples:
+ data = torch.zeros(1, 8_000)
+
+ class _AudioDecoder:
+ def __init__(self, _path):
+ self.metadata = _Metadata()
+
+ def get_samples_played_in_range(self, start, stop):
+ ranges.append((start, stop))
+ return _Samples()
+
+ monkeypatch.setitem(sys.modules, "torchcodec", types.SimpleNamespace(decoders = None))
+ monkeypatch.setitem(
+ sys.modules,
+ "torchcodec.decoders",
+ types.SimpleNamespace(AudioDecoder = _AudioDecoder),
+ )
+ waveform, rate = inference_route._decode_audio_with_torchcodec("ignored")
+ assert rate == 16_000
+ assert waveform.shape[-1] == 8_000
+ assert len(ranges) == 1
+ start, stop = ranges[0]
+ assert start == 0.0
+ assert stop <= inference_route._MAX_AUDIO_SECONDS + 1
+
+
+def test_the_last_resort_decoder_refuses_an_overlong_file(monkeypatch):
+ class _Metadata:
+ sample_rate = 16_000
+ num_channels = 1
+ duration_seconds = inference_route._MAX_AUDIO_SECONDS + 60
+
+ class _AudioDecoder:
+ def __init__(self, _path):
+ self.metadata = _Metadata()
+
+ def get_samples_played_in_range(self, _start, _stop):
+ raise AssertionError("an over-limit file must not be decoded")
+
+ monkeypatch.setitem(sys.modules, "torchcodec", types.SimpleNamespace(decoders = None))
+ monkeypatch.setitem(
+ sys.modules,
+ "torchcodec.decoders",
+ types.SimpleNamespace(AudioDecoder = _AudioDecoder),
+ )
+ try:
+ inference_route._decode_audio_with_torchcodec("ignored")
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected the duration limit to refuse this file")
+
+
+def test_the_last_resort_decoder_refuses_without_torchcodec(monkeypatch):
+ """No bounded reader means no read, not an unbounded one."""
+ monkeypatch.setitem(sys.modules, "torchcodec", None)
+ monkeypatch.setitem(sys.modules, "torchcodec.decoders", None)
+ try:
+ inference_route._decode_audio_with_torchcodec("ignored")
+ except RuntimeError as error:
+ assert "could not be decoded" in str(error)
+ else:
+ raise AssertionError("expected a refusal with no bounded reader")
+
+
+def test_a_free_format_frame_does_not_authorise_a_passthrough():
+ """Bitrate index 0 carries no length in the header, so the walk cannot
+ establish one and the file has to be bounded by decoding it instead."""
+ free = bytes([0xFF, 0xE0 | (0x00 << 3) | (0x01 << 1) | 0x01, 0x00, 0x00])
+ assert inference_route._mp3_seconds(free + _mp3_frames(5), 60.0) is None
+
+
+def test_a_forged_byte_rate_cannot_shorten_a_forwarded_wav(monkeypatch):
+ """nAvgBytesPerSec is redundant with the fields around it, so it is the one
+ that can be moved alone. Multiplied by ten thousand it made a long recording
+ read as a fraction of a second, and a short file is forwarded untouched."""
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)
+ payload = b"\x80" * (8_000 * 2)
+ honest = _wav_header(8_000, 1, 8, len(payload)) + payload
+ assert inference_route._wav_seconds(honest) > 1
+
+ forged = _wav_header(8_000, 1, 8, len(payload), byte_rate = 8_000 * 10_000) + payload
+ assert inference_route._wav_seconds(forged) == inference_route._wav_seconds(honest)
+ try:
+ inference_route._prepare_audio_for_llama(base64.b64encode(forged).decode())
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("a forged byte rate walked past the duration limit")
+
+
+def test_an_id3v1_trailer_does_not_hide_the_duration():
+ """A trailing tag is metadata; the frames before it still count."""
+ tagged = _mp3_frames(10) + b"TAG" + b"\x00" * 125
+ seconds = inference_route._mp3_seconds(tagged, 60.0)
+ assert seconds is not None
+ assert 9.0 < seconds < 11.0
+
+
+def test_a_switch_preflight_answers_an_overlong_upload_like_the_serving_path(monkeypatch):
+ """The same file must not read as 413 when the model is loaded and 400 when a
+ swap happens to be running."""
+ import asyncio
+
+ from fastapi import HTTPException
+
+ def _too_long(_b64):
+ raise inference_route._DecodedAudioTooLongError("decoded audio exceeds the limit")
+
+ monkeypatch.setattr(inference_route, "_prepare_audio_for_llama", _too_long)
+ monkeypatch.setattr(inference_route, "_decode_audio_base64", _too_long)
+ monkeypatch.setattr(inference_route, "_audio_decoder_is_available", lambda: True)
+
+ for target_is_gguf in (True, False):
+ try:
+ asyncio.run(inference_route._preflight_audio_for_switch({"b64": "x"}, target_is_gguf))
+ except HTTPException as error:
+ assert error.status_code == 413, target_is_gguf
+ assert error.detail == inference_route._audio_too_long_detail()
+ else:
+ raise AssertionError(f"expected a 413 for target_is_gguf={target_is_gguf}")
+
+
+def test_a_switch_preflight_still_reports_undecodable_audio_as_a_bad_value(monkeypatch):
+ """Only the duration case moves; a file that really cannot be read keeps 400."""
+ import asyncio
+
+ from fastapi import HTTPException
+
+ def _broken(_b64):
+ raise ValueError("not audio at all")
+
+ monkeypatch.setattr(inference_route, "_prepare_audio_for_llama", _broken)
+ monkeypatch.setattr(inference_route, "_decode_audio_base64", _broken)
+ monkeypatch.setattr(inference_route, "_audio_decoder_is_available", lambda: True)
+
+ for target_is_gguf in (True, False):
+ try:
+ asyncio.run(inference_route._preflight_audio_for_switch({"b64": "x"}, target_is_gguf))
+ except HTTPException as error:
+ assert error.status_code == 400, target_is_gguf
+ else:
+ raise AssertionError(f"expected a 400 for target_is_gguf={target_is_gguf}")
+
+
+def test_every_path_names_the_limit_the_same_way(monkeypatch):
+ """One message, so a client sees one answer wherever the limit was found."""
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 90 * 60)
+ assert inference_route._audio_too_long_detail() == "Audio is too long (max 90 minutes)."
+ # Explicitly UTF-8: read_text() follows the locale encoding, which is cp1252
+ # on a stock Windows runner, and the module holds bytes cp1252 cannot decode.
+ source = (pathlib.Path(inference_route.__file__)).read_text(encoding = "utf-8")
+ assert source.count('f"Audio is too long') == 1
+
+
+def _old_resample(arr, source_rate, target_rate):
+ """The whole-array pass this replaced, kept so the output can be compared."""
+ duration = len(arr) / float(source_rate)
+ target_len = max(1, int(round(duration * target_rate)))
+ source_x = np.linspace(0.0, duration, num = len(arr), endpoint = False)
+ target_x = np.linspace(0.0, duration, num = target_len, endpoint = False)
+ return np.interp(target_x, source_x, arr).astype(np.float32)
+
+
+ ("count", "source_rate", "target_rate"),
+ [
+ (48_000, 48_000, 16_000),
+ (2_500_000, 48_000, 47_999),
+ ((1 << 20) + 1, 48_000, 44_100),
+ (100_000, 16_000, 48_000),
+ (3, 8_000, 4_000),
+ ],
+)
+def test_the_sliced_resampler_returns_what_the_whole_array_pass_did(
+ count, source_rate, target_rate
+):
+ """Slicing is a memory change, not an audio one: same samples, same dtype."""
+ arr = np.random.default_rng(11).standard_normal(count).astype(np.float32)
+ fitted = inference_route._resample_mono_linear(arr, source_rate, target_rate)
+ expected = _old_resample(arr, source_rate, target_rate)
+ assert fitted.dtype == expected.dtype
+ assert np.array_equal(fitted, expected)
+
+
+def test_resampling_does_not_allocate_grids_the_length_of_the_recording():
+ """The two float64 grids were 8 bytes per input sample each, so a 30-minute
+ upload spent more on the coordinates than on the audio."""
+ import tracemalloc
+
+ arr = np.zeros(4_000_000, dtype = np.float32)
+ tracemalloc.start()
+ try:
+ tracemalloc.reset_peak()
+ before = tracemalloc.get_traced_memory()[0]
+ fitted = inference_route._resample_mono_linear(arr, 48_000, 16_000)
+ peak = tracemalloc.get_traced_memory()[1] - before
+ finally:
+ tracemalloc.stop()
+ # The result is a third of the input; anything past it plus a slice means a
+ # grid was materialized whole.
+ assert peak < fitted.nbytes + 16 * 1024 * 1024
+
+
+def test_the_wav_encoder_writes_the_bytes_it_always_wrote():
+ """In place beyond the first copy, and the same file out."""
+ rng = np.random.default_rng(3)
+ for scale in (0.5, 3.0):
+ arr = (rng.standard_normal(5_000) * scale).astype(np.float32)
+ arr[7], arr[8], arr[9] = np.nan, np.inf, -np.inf
+ scrubbed = np.nan_to_num(
+ np.asarray(arr, dtype = np.float32).flatten(), posinf = 0.0, neginf = 0.0
+ )
+ peak = float(np.abs(scrubbed).max())
+ if peak > 1.0:
+ scrubbed = scrubbed / peak
+ expected = (scrubbed * 32767.0).astype(np.int16).tobytes()
+ written = inference_route._mono_f32_to_wav_bytes(arr, 16_000)
+ assert written[inference_route._WAV_HEADER_BYTES :] == expected
+ assert len(written) == inference_route._WAV_HEADER_BYTES + arr.size * 2
+
+
+def test_the_wav_encoder_leaves_its_caller_s_array_alone():
+ """Scaling in place must happen on the copy, not on the decoded audio."""
+ arr = np.array([0.5, -2.0, 0.25], dtype = np.float32)
+ inference_route._mono_f32_to_wav_bytes(arr, 8_000)
+ assert np.array_equal(arr, np.array([0.5, -2.0, 0.25], dtype = np.float32))
+
+
+class _CountingSoundFile:
+ """A libsndfile stand-in whose header count can disagree with what it yields."""
+
+ samplerate = 8_000
+ channels = 1
+ blocks_yielded = 4
+ declared_frames = 4
+ block_len = 1_000
+ allocations: list = []
+
+ def __init__(self, _source):
+ self.frames = type(self).declared_frames * type(self).block_len
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def blocks(self, **_kwargs):
+ for index in range(type(self).blocks_yielded):
+ yield np.full(type(self).block_len, float(index), dtype = np.float32)
+
+
+def _decode_with(monkeypatch, **attributes):
+ for name, value in attributes.items():
+ setattr(_CountingSoundFile, name, value)
+ monkeypatch.setitem(
+ sys.modules, "soundfile", types.SimpleNamespace(SoundFile = _CountingSoundFile)
+ )
+ joins = []
+ real_concatenate = np.concatenate
+
+ def counted(arrays, *args, **kwargs):
+ joins.append(len(list(arrays)) if isinstance(arrays, list) else 1)
+ return real_concatenate(arrays, *args, **kwargs)
+
+ monkeypatch.setattr(np, "concatenate", counted)
+ arr, rate = inference_route._decode_audio_mono(b"blocks")
+ return arr, rate, joins
+
+
+def test_an_honest_header_decodes_into_one_array(monkeypatch):
+ """Holding every block and then joining them copied the whole recording twice."""
+ arr, rate, joins = _decode_with(
+ monkeypatch, blocks_yielded = 4, declared_frames = 4, block_len = 1_000
+ )
+ assert rate == 8_000
+ assert joins == []
+ assert len(arr) == 4_000
+ assert np.array_equal(arr[::1_000], np.array([0.0, 1.0, 2.0, 3.0], dtype = np.float32))
+
+
+def test_a_header_claiming_more_than_it_holds_buys_no_buffer(monkeypatch):
+ """A buffer sized from an unread header is how a small file asks for a large
+ allocation, so the count is only trusted once the decode has met half of it."""
+ sizes = []
+ real_empty = np.empty
+
+ def watched(shape, *args, **kwargs):
+ if isinstance(shape, int):
+ sizes.append(shape)
+ return real_empty(shape, *args, **kwargs)
+
+ monkeypatch.setattr(np, "empty", watched)
+ arr, _rate, _joins = _decode_with(
+ monkeypatch, blocks_yielded = 1, declared_frames = 200, block_len = 1_000
+ )
+ assert len(arr) == 1_000
+ assert all(size <= 2_000 for size in sizes), sizes
+
+
+def test_a_header_that_undercounts_still_returns_every_sample(monkeypatch):
+ """The buffer grows past a short header rather than spilling into a list.
+
+ Collecting the overflow and joining it at the end allocated a second copy of
+ the whole recording, which is what the growing buffer exists to avoid, so a
+ short header must cost a resize and never a final join."""
+ arr, _rate, joins = _decode_with(
+ monkeypatch, blocks_yielded = 5, declared_frames = 3, block_len = 1_000
+ )
+ assert len(arr) == 5_000
+ assert joins == []
+ assert np.array_equal(arr[::1_000], np.array([0.0, 1.0, 2.0, 3.0, 4.0], dtype = np.float32))
+
+
+def test_a_header_past_the_duration_cap_is_not_preallocated(monkeypatch):
+ """The cap refuses the audio anyway; it must not buy that much array first.
+
+ The buffer still grows with what has actually been decoded, so the guarantee
+ is not that nothing is allocated but that nothing the header claims is: every
+ allocation stays inside the ceiling the decode is about to refuse at."""
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)
+ sizes = []
+ real_empty = np.empty
+
+ def watched(shape, *args, **kwargs):
+ if isinstance(shape, int):
+ sizes.append(shape)
+ return real_empty(shape, *args, **kwargs)
+
+ monkeypatch.setattr(np, "empty", watched)
+ try:
+ _decode_with(monkeypatch, blocks_yielded = 4, declared_frames = 4, block_len = 3_000)
+ except inference_route._DecodedAudioTooLongError:
+ pass
+ else:
+ raise AssertionError("expected the duration cap to refuse this decode")
+ # 8 kHz against a one-second cap, so the ceiling is 8000 and the header's
+ # 12000 is well past it.
+ ceiling = inference_route._decoded_sample_ceiling(8_000)
+ assert sizes, "the decoder allocated nothing at all"
+ assert max(sizes) <= ceiling, sizes
+
+
+def test_a_header_that_undercounts_keeps_the_samples_in_order(monkeypatch):
+ """The buffer is closed by the first block that does not fit it.
+
+ Re-testing the fit for every block let a short final block drop back into
+ the unused tail and be emitted ahead of the blocks that overflowed before
+ it. The length stayed right, so nothing downstream noticed, and what
+ reached the model was the same audio with a piece of it moved.
+ """
+
+ class _VaryingSoundFile:
+ samplerate = 8_000
+ channels = 1
+ frames = 2_500
+
+ def __init__(self, _source):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def blocks(self, **_kwargs):
+ at = 0
+ for length in (1_000, 1_000, 1_000, 500):
+ yield np.arange(at, at + length, dtype = np.float32)
+ at += length
+
+ monkeypatch.setitem(
+ sys.modules, "soundfile", types.SimpleNamespace(SoundFile = _VaryingSoundFile)
+ )
+ arr, rate = inference_route._decode_audio_mono_with_soundfile(b"undercounted")
+ assert rate == 8_000
+ # A ramp in, so a reorder is a descent out.
+ assert np.array_equal(arr, np.arange(3_500, dtype = np.float32))
+
+
+def test_torchaudio_alone_can_still_decode_audio(monkeypatch):
+ """Backwards compatibility for an install that predates PyAV.
+
+ Before the bounded readers existed this path called torchaudio.load()
+ outright, so torchaudio on its own was enough. torchaudio 2.9 removed
+ info(), which is what sends such an install to the streaming chain, and
+ that chain needs libsndfile, PyAV or librosa. Losing audio on upgrade is
+ not an acceptable way to gain a memory bound, so it falls back to the
+ bounded torchcodec reader that 2.9's own load() decodes through.
+ """
+ # torch is only a source of fake tensors for the fake torchaudio below, so
+ # a machine without it should skip rather than fail, as the av cases do.
+ torch = pytest.importorskip("torch")
+
+ class _Metadata:
+ sample_rate = 16_000
+ num_channels = 1
+ duration_seconds = 0.5
+
+ class _Samples:
+ data = torch.zeros(1, 8_000)
+
+ class _AudioDecoder:
+ def __init__(self, _path):
+ self.metadata = _Metadata()
+
+ def get_samples_played_in_range(self, _start, _stop):
+ return _Samples()
+
+ class _FakeTorchaudio:
+ class transforms:
+ pass
+
+ monkeypatch.setitem(sys.modules, "torchaudio", _FakeTorchaudio)
+ monkeypatch.setitem(sys.modules, "torchcodec", types.SimpleNamespace(decoders = None))
+ monkeypatch.setitem(
+ sys.modules,
+ "torchcodec.decoders",
+ types.SimpleNamespace(AudioDecoder = _AudioDecoder),
+ )
+ for absent in ("soundfile", "av", "librosa"):
+ monkeypatch.setitem(sys.modules, absent, None)
+
+ payload = b"\x80" * 8_000
+ raw = _wav_header(8_000, 1, 8, len(payload)) + payload
+ decoded = inference_route._decode_audio_base64(base64.b64encode(raw).decode())
+ assert decoded.shape[0] == 8_000
+
+
+def test_a_tag_in_the_middle_cannot_shorten_a_concatenated_mp3(monkeypatch):
+ """`cat one.mp3 two.mp3 > both.mp3` is how people join MP3s, and it leaves
+ the first file's 128-byte ID3v1 tag sitting between the two streams.
+
+ Matching a trailer by its magic alone accepted that tag as the end of the
+ recording, so an hour of audio behind it reported the first file's few
+ seconds and was forwarded untouched. A trailer now has to account for its
+ own length and reach EOF, and one that does not means the walk never read
+ the whole file.
+ """
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 60)
+ joined = _mp3_frames(5) + b"TAG" + bytes(125) + _mp3_frames(300)
+ assert inference_route._mp3_seconds(joined, 60.0) is None
+
+ _decode_instead_of_forwarding(monkeypatch)
+ _encoded, container = inference_route._prepare_audio_for_llama(
+ base64.b64encode(joined).decode()
+ )
+ assert container == "wav"
+
+
+def test_a_trailer_that_reaches_eof_still_ends_the_count():
+ """The counterpart: real trailers must not cost a needless transcode."""
+ frames = _mp3_frames(10)
+ ape_size = 64
+ ape = (
+ b"APETAGEX"
+ + (2000).to_bytes(4, "little")
+ + ape_size.to_bytes(4, "little")
+ + (1).to_bytes(4, "little")
+ + (0x80000000).to_bytes(4, "little")
+ + bytes(8)
+ + bytes(ape_size)
+ )
+ for trailer in (b"", b"TAG" + bytes(125), ape, b"TAG" + bytes(125) + ape):
+ seconds = inference_route._mp3_seconds(frames + trailer, 60.0)
+ assert seconds is not None and 9.9 < seconds < 10.1, len(trailer)
+
+
+def test_pyav_writes_into_one_buffer_rather_than_collecting_blocks(monkeypatch):
+ """Dropping each block after copying it does not bound anything.
+
+ The blocks are small, so freeing them returns them to the allocator's free
+ lists and not to the OS: resident memory for a 30-minute 48 kHz upload
+ measured 2.0x the waveform whether the join was np.concatenate or a fill
+ loop. Only writing each block into the output as it arrives holds one copy,
+ so the decoder must never build a list of the whole decode.
+ """
+ raw = _encode_amr()
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", None)
+ expected, rate = inference_route._decode_audio_mono(raw)
+
+ # An under-reported duration has to grow the buffer instead of appending to
+ # a list, and the samples must survive the growth copies in decode order.
+ monkeypatch.setattr(inference_route, "_av_expected_samples", lambda *_args: 1)
+ grown, grown_rate = inference_route._decode_audio_mono(raw)
+ assert grown_rate == rate
+ assert np.array_equal(grown, expected)
+
+
+def test_a_forged_container_duration_cannot_ask_for_a_huge_buffer(monkeypatch):
+ """The duration only sizes the first allocation, so a lie costs a growth
+ copy, never a wrong result and never an allocation the decode would refuse."""
+
+ class _Stream:
+ duration = None
+ time_base = None
+
+ class _Container:
+ duration = 10**18
+ streams = types.SimpleNamespace(audio = [_Stream()])
+
+ ceiling = inference_route._decoded_sample_ceiling(48_000)
+ assert inference_route._av_expected_samples(_Container(), 48_000, ceiling) == ceiling
+
+ class _Undeclared(_Container):
+ duration = 0
+
+ # Nothing declared starts at a minute and grows from there.
+ assert inference_route._av_expected_samples(_Undeclared(), 48_000, ceiling) == 60 * 48_000 + 1
+
+
+def test_no_allocation_outgrows_the_limit_the_decode_enforces(monkeypatch):
+ """Both limits bind at once, and the smaller one is where the decode stops.
+
+ Sizing an allocation by only one of them let an 8 kHz file declaring three
+ hours reserve 86.4M floats (330 MB) on the way to being refused at 14.4M
+ (55 MB), and the doubling that covers an under-reported duration had no
+ ceiling at all: a 48 kHz file declaring twenty minutes and holding
+ twenty-one grew to 115.2M against an 86.4M limit, with the old buffer still
+ live for the copy.
+ """
+
+ class _Stream:
+ duration = None
+ time_base = None
+
+ class _Container:
+ duration = 3 * 3600 * 10**6
+ streams = types.SimpleNamespace(audio = [_Stream()])
+
+ # The clock binds below the sample ceiling at 8 kHz, and vice versa at 48.
+ assert inference_route._decoded_sample_ceiling(8_000) == 8_000 * 1800
+ assert inference_route._decoded_sample_ceiling(192_000) == inference_route._MAX_DECODED_SAMPLES
+
+ ceiling = inference_route._decoded_sample_ceiling(8_000)
+ assert inference_route._av_expected_samples(_Container(), 8_000, ceiling) == ceiling
+
+ # And the growth path stops there too rather than doubling past it.
+ raw = _encode_amr()
+ monkeypatch.setitem(sys.modules, "soundfile", None)
+ monkeypatch.setitem(sys.modules, "librosa", None)
+ sizes: list[int] = []
+ real_empty = np.empty
+
+ def record(shape, *args, **kwargs):
+ if isinstance(shape, int):
+ sizes.append(shape)
+ return real_empty(shape, *args, **kwargs)
+
+ monkeypatch.setattr(np, "empty", record)
+ monkeypatch.setattr(inference_route, "_av_expected_samples", lambda *_args: 1)
+ samples, rate = inference_route._decode_audio_mono(raw)
+ assert len(samples) > 0
+ assert sizes, "the decoder allocated nothing"
+ assert max(sizes) <= inference_route._decoded_sample_ceiling(rate)
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 3b55df33214..5e9b4fe558f 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -150,6 +150,7 @@ import {
} from "@/features/chat/utils/continuation";
import { holdAutoContinueRun } from "@/features/chat/utils/auto-continue-run-keeper";
import { McpComposerButton } from "@/features/chat/mcp-composer-button";
+import { pickerAcceptForTextBasenames } from "@/features/chat/text-attachment-accept";
import {
COMPOSER_INPUT_SELECTOR,
isSurfaceInForeground,
@@ -5741,18 +5742,18 @@ const ToolStatusDisplay: FC = () => {
// Plus menu: attachment and workflow actions. Opens downward in the welcome
// composer; the docked composer passes side="top" to open upward.
const AUDIO_ACCEPT_TOKEN_RE =
- /^(audio\/|\.(?:wav|mp3|m4a|ogg|oga|flac)$)/i;
+ /^(audio\/|\.(?:wav|mp3|mp2|m4a|ogg|oga|opus|flac|aac|aiff|aif|aifc|caf|wma|amr)$)/i;
function attachmentAcceptForPicker(accept: string, audioEnabled: boolean): string {
- if (audioEnabled || accept === "*") {
- return accept;
- }
- const filtered = accept
- .split(",")
- .map((token) => token.trim())
- .filter((token) => token && !AUDIO_ACCEPT_TOKEN_RE.test(token))
- .join(",");
- return filtered || accept;
+ const enabledAccept =
+ audioEnabled || accept === "*"
+ ? accept
+ : accept
+ .split(",")
+ .map((token) => token.trim())
+ .filter((token) => token && !AUDIO_ACCEPT_TOKEN_RE.test(token))
+ .join(",") || accept;
+ return pickerAcceptForTextBasenames(enabledAccept);
}
const ComposerToolsMenu: FC<{
diff --git a/studio/frontend/src/features/chat/attachment-content.ts b/studio/frontend/src/features/chat/attachment-content.ts
index bf7601539ef..5f6bb876f80 100644
--- a/studio/frontend/src/features/chat/attachment-content.ts
+++ b/studio/frontend/src/features/chat/attachment-content.ts
@@ -2,7 +2,11 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { type Unzipped, strFromU8, unzipSync, zipSync } from "fflate";
-import { TEXT_ATTACHMENT_ACCEPT } from "./text-attachment-accept";
+import {
+ TEXT_ATTACHMENT_ACCEPT,
+ decodeTextAttachmentBytes,
+ isTextAttachmentName,
+} from "./text-attachment-accept";
import {
MAX_OPEN_DOCUMENT_ARCHIVE_BYTES,
@@ -26,7 +30,8 @@ export type AttachmentText = {
truncated: boolean;
};
-const AUDIO_ATTACHMENT_RE = /\.(wav|mp3|m4a|ogg|oga|flac|webm|mp4|aac)$/i;
+const AUDIO_ATTACHMENT_RE =
+ /\.(wav|mp3|mp2|m4a|ogg|oga|opus|flac|webm|mp4|aac|aiff|aif|aifc|caf|wma|amr)$/i;
const AUDIO_MIME_RE = /^audio\//i;
const PDF_ATTACHMENT_RE = /\.pdf$/i;
const DOCX_ATTACHMENT_RE = /\.docx$/i;
@@ -97,12 +102,20 @@ const MAX_DOCX_UNPACKED_BYTES = 2 * MAX_OPEN_DOCUMENT_ARCHIVE_BYTES;
const AUDIO_EXTENSION_MIMES: Record<string, string> = {
wav: "audio/wav",
mp3: "audio/mpeg",
+ mp2: "audio/mpeg",
m4a: "audio/mp4",
mp4: "audio/mp4",
ogg: "audio/ogg",
oga: "audio/ogg",
+ opus: "audio/opus",
flac: "audio/flac",
aac: "audio/aac",
+ aiff: "audio/aiff",
+ aif: "audio/aiff",
+ aifc: "audio/aiff",
+ caf: "audio/x-caf",
+ wma: "audio/x-ms-wma",
+ amr: "audio/amr",
webm: "audio/webm",
};
// Node.TEXT_NODE and Node.ELEMENT_NODE, spelled out so the extractor runs where the DOM globals do not.
@@ -149,78 +162,245 @@ const HTML_BLOCK_TAGS = new Set([
]);
/** The extensions the text and html previews colour, mapped to shiki ids; one missing here previews unstyled. */
const CODE_ATTACHMENT_LANGUAGES: Record<string, string> = {
+ ada: "ada",
+ adb: "ada",
+ adoc: "asciidoc",
+ ads: "ada",
+ applescript: "applescript",
+ asciidoc: "asciidoc",
+ asm: "asm",
astro: "astro",
+ avsc: "json",
+ awk: "awk",
bash: "shellscript",
bat: "bat",
+ bazel: "python",
+ bib: "bibtex",
+ bicep: "bicep",
+ bzl: "python",
c: "c",
+ cabal: "haskell",
+ cairo: "cairo",
+ cbl: "cobol",
cc: "cpp",
cfg: "ini",
cjs: "javascript",
+ cl: "lisp",
+ clj: "clojure",
+ cljc: "clojure",
+ cljs: "clojure",
+ cls: "latex",
cmake: "cmake",
+ cmd: "bat",
+ cnf: "ini",
+ cob: "cobol",
+ comp: "glsl",
conf: "ini",
+ containerfile: "docker",
cpp: "cpp",
+ cql: "sql",
+ cr: "crystal",
cs: "csharp",
+ csh: "shellscript",
+ cshtml: "razor",
+ csproj: "xml",
css: "css",
+ cts: "typescript",
+ cu: "cuda",
+ cuh: "cuda",
cxx: "cpp",
+ d: "d",
dart: "dart",
+ desktop: "ini",
+ dhall: "dhall",
diff: "diff",
dockerfile: "docker",
+ dot: "dot",
+ edn: "clojure",
+ ejs: "ejs",
+ el: "lisp",
env: "dotenv",
+ erb: "erb",
+ erl: "erlang",
+ ex: "elixir",
+ exs: "elixir",
+ f: "fortran-free-form",
+ f03: "fortran-free-form",
+ f90: "fortran-free-form",
+ f95: "fortran-free-form",
+ feature: "gherkin",
fish: "fish",
+ for: "fortran-free-form",
+ frag: "glsl",
+ fs: "fsharp",
+ fsi: "fsharp",
+ fsproj: "xml",
+ fsx: "fsharp",
+ gd: "gdscript",
+ gemspec: "ruby",
+ geojson: "json",
+ geom: "glsl",
+ glsl: "glsl",
+ gn: "python",
+ gni: "python",
go: "go",
gql: "graphql",
gradle: "groovy",
graphql: "graphql",
+ groovy: "groovy",
+ gv: "dot",
h: "c",
+ haml: "haml",
+ handlebars: "handlebars",
+ har: "json",
+ hbs: "handlebars",
+ hcl: "hcl",
+ hh: "cpp",
+ hlsl: "hlsl",
hpp: "cpp",
+ hql: "sql",
+ hrl: "erlang",
+ hs: "haskell",
htm: "html",
html: "html",
+ http: "http",
+ hx: "haxe",
+ hxx: "cpp",
ini: "ini",
+ inl: "cpp",
+ ipp: "cpp",
+ j2: "jinja",
+ jade: "pug",
java: "java",
+ jinja: "jinja",
+ jinja2: "jinja",
jl: "julia",
js: "javascript",
json: "json",
+ json5: "json5",
+ jsonc: "jsonc",
jsonl: "json",
jsx: "jsx",
+ ksh: "shellscript",
kt: "kotlin",
kts: "kotlin",
+ latex: "latex",
less: "less",
+ lhs: "haskell",
+ liquid: "liquid",
+ lisp: "lisp",
+ lsp: "lisp",
lua: "lua",
+ mak: "make",
makefile: "make",
- md: "markdown",
markdown: "markdown",
+ md: "markdown",
mdx: "mdx",
+ mermaid: "mermaid",
+ metal: "cpp",
mjs: "javascript",
+ mk: "make",
+ ml: "ocaml",
+ mli: "ocaml",
+ mmd: "mermaid",
+ mojo: "mojo",
+ move: "move",
+ mts: "typescript",
+ mustache: "handlebars",
ndjson: "json",
+ nim: "nim",
+ nimble: "nim",
+ nims: "nim",
+ ninja: "ninja",
+ nix: "nix",
+ njk: "jinja",
+ nu: "nushell",
+ pas: "pascal",
php: "php",
pl: "perl",
+ plantuml: "plantuml",
+ plist: "xml",
+ plsql: "sql",
pm: "perl",
+ podspec: "ruby",
+ pp: "pascal",
+ prisma: "prisma",
properties: "ini",
+ props: "xml",
proto: "proto",
ps1: "powershell",
+ psd1: "powershell",
+ psm1: "powershell",
+ psql: "sql",
+ psv: "csv",
+ pug: "pug",
+ puml: "plantuml",
+ pxd: "python",
py: "python",
pyi: "python",
+ pyx: "python",
+ qmd: "markdown",
+ qml: "qml",
r: "r",
+ rake: "ruby",
+ razor: "razor",
rb: "ruby",
+ reg: "ini",
+ rest: "http",
+ resx: "xml",
+ rkt: "racket",
+ rmd: "markdown",
+ ron: "rust",
rs: "rust",
rst: "rst",
+ s: "asm",
+ sas: "sas",
sass: "sass",
+ sbt: "scala",
scala: "scala",
+ scm: "scheme",
scss: "scss",
+ service: "ini",
sh: "shellscript",
+ shader: "hlsl",
+ sol: "solidity",
sql: "sql",
+ ss: "scheme",
+ star: "python",
+ starlark: "python",
+ sty: "latex",
+ styl: "stylus",
+ sv: "system-verilog",
svelte: "svelte",
svg: "xml",
+ svh: "system-verilog",
swift: "swift",
+ targets: "xml",
+ tcl: "tcl",
+ tcsh: "shellscript",
+ tex: "latex",
tf: "terraform",
+ tfstate: "json",
tfvars: "terraform",
toml: "toml",
ts: "typescript",
tsx: "tsx",
+ twig: "twig",
+ v: "v",
+ vb: "vb",
+ vbproj: "xml",
+ vbs: "vb",
+ vert: "glsl",
+ vhd: "vhdl",
+ vhdl: "vhdl",
vue: "vue",
+ wgsl: "wgsl",
+ xlf: "xml",
+ xliff: "xml",
xml: "xml",
yaml: "yaml",
yml: "yaml",
+ zig: "zig",
zsh: "shellscript",
};
// Long attachments still have to render inside a dialog, so the preview stops
@@ -679,12 +859,37 @@ export async function readAttachmentText(
return { label: null, ...(await readBoundedText(file)) };
}
+// Formats that state their own encoding somewhere other than the first bytes: a
+// gettext header sits below the translator comments, and a mail or vCard
+// declaration below whatever came before it in the archive.
+const DECLARES_ITS_CHARSET_RE = /\.(?:po|pot|eml|mbox|vcf)$/i;
+
async function readBoundedText(
file: File,
): Promise<{ text: string; truncated: boolean }> {
const truncated = file.size > MAX_PREVIEW_TEXT_BYTES;
const slice = truncated ? file.slice(0, MAX_PREVIEW_TEXT_BYTES) : file;
- return { text: await slice.text(), truncated };
+ // Strict decoding belongs to the files the text adapter owns, where refusing
+ // is better than sending mojibake to the model. A preview of someone else's
+ // file may not be stricter than the adapter that accepted it: .html goes to
+ // the HTML adapter, which sends a windows-1252 page happily, and reading the
+ // preview through the strict path meant opening one threw where it used to
+ // render.
+ if (!isTextAttachmentName(file.name)) {
+ return { text: await slice.text(), truncated };
+ }
+ const bytes = new Uint8Array(await slice.arrayBuffer());
+ // The declaration can sit past the preview slice, and looking for it inside
+ // the slice reported an error for a file the attachment itself decodes. Only
+ // the formats that can carry one that far in pay for the second read.
+ const whole =
+ truncated && DECLARES_ITS_CHARSET_RE.test(file.name)
+ ? new Uint8Array(await file.arrayBuffer())
+ : bytes;
+ return {
+ text: decodeTextAttachmentBytes(bytes, file.name, truncated, whole),
+ truncated,
+ };
}
// A sent attachment keeps only the text its adapter produced, so the preview
diff --git a/studio/frontend/src/features/chat/audio-attachment-adapter.ts b/studio/frontend/src/features/chat/audio-attachment-adapter.ts
index d11290bf2dc..f082a949abb 100644
--- a/studio/frontend/src/features/chat/audio-attachment-adapter.ts
+++ b/studio/frontend/src/features/chat/audio-attachment-adapter.ts
@@ -2,7 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import {
- AUDIO_ACCEPT,
+ AUDIO_ATTACHMENT_ACCEPT,
fileToBase64,
getAudioSizeError,
} from "@/lib/audio-utils";
@@ -29,8 +29,11 @@ function newAttachmentId(): string {
export class AudioAttachmentAdapter implements AttachmentAdapter {
// MIME is unreliable for some containers (m4a), so also match by
// extension. No .webm extension: it would claim video/webm files; real
- // audio webm (MediaRecorder) always reports the audio/webm MIME.
- accept = `${AUDIO_ACCEPT},audio/x-m4a,.wav,.mp3,.m4a,.ogg,.oga,.flac`;
+ // audio webm (MediaRecorder) always reports the audio/webm MIME. .mp4 and
+ // .m4v stay off for the same reason; only .m4a is audio-only. Not the picker
+ // list: this decides routing, and .3gp is in that list only so a dialog can
+ // offer a recording.
+ accept = AUDIO_ATTACHMENT_ACCEPT;
private readonly attachmentIds = new Set<string>();
async add({ file }: { file: File }): Promise<PendingAttachment> {
diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx
index 1b474a9d71b..21fcc0e8255 100644
--- a/studio/frontend/src/features/chat/runtime-provider.tsx
+++ b/studio/frontend/src/features/chat/runtime-provider.tsx
@@ -2,6 +2,10 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { authFetch } from "@/features/auth";
+import {
+ classifiedAttachmentFile,
+ needsAttachmentTrackInspection,
+} from "@/lib/video-utils";
import {
AssistantRuntimeProvider,
type Attachment,
@@ -59,6 +63,16 @@ import {
getDocxAttachmentError,
} from "./attachment-content";
import { AudioAttachmentAdapter } from "./audio-attachment-adapter";
+import {
+ isBinaryPropertyList,
+ isBinaryTrackerModule,
+ MAX_TEXT_ATTACHMENT_BYTES,
+ isBinaryOfficeTemplate,
+ isCompiledFortranModule,
+ isBinaryVobSubSubtitle,
+ readTextAttachmentOnce,
+ UndecodableTextError,
+} from "./text-attachment-accept";
import {
loadConnectionsEnabled,
loadExternalProviders,
@@ -210,7 +224,31 @@ class PreStreamAwareAttachmentAdapter implements AttachmentAdapter {
}
add(state: { file: File }) {
- return this.delegate.add(state);
+ // A composite picks its adapter synchronously from the name and MIME type,
+ // and both say "video" for an audio-only 3GP recording, so settle that from
+ // the container's own tracks first, as the native readers do. Every other
+ // file goes straight through, keeping the delegate's own return.
+ if (!needsAttachmentTrackInspection(state.file)) {
+ return this.delegate.add(state);
+ }
+ return this.addInspected(state);
+ }
+
+ private async addInspected(state: {
+ file: File;
+ }): Promise<PendingAttachment> {
+ const file = await classifiedAttachmentFile(state.file);
+ const added = await this.delegate.add({ ...state, file });
+ if (Symbol.asyncIterator in added) {
+ // Only the audio and video adapters claim a 3GP and both resolve to one
+ // attachment, so this drains a generator to its last value rather than
+ // forwarding the progress an adapter here does not report.
+ let last: PendingAttachment | undefined;
+ for await (const value of added) last = value;
+ if (!last) throw new Error("The attachment adapter yielded nothing.");
+ return last;
+ }
+ return added;
}
remove(attachment: Attachment): Promise<void> {
@@ -370,6 +408,55 @@ class TextAttachmentAdapter implements AttachmentAdapter {
accept = TEXT_ATTACHMENT_ACCEPT;
async add({ file }: { file: File }): Promise<PendingAttachment> {
+ // Before any read: decoding an .mbox of arbitrary size would hold the bytes
+ // and the decoded string at once, and the native path already stops here.
+ if (file.size > MAX_TEXT_ATTACHMENT_BYTES) {
+ const reason = `Text attachments are limited to ${
+ MAX_TEXT_ATTACHMENT_BYTES / (1024 * 1024)
+ } MB.`;
+ toast.error(reason);
+ throw new Error(reason);
+ }
+ if (await isBinaryPropertyList(file)) {
+ const reason =
+ "Binary property-list files aren't supported. Convert the file to text before attaching it.";
+ toast.error(reason);
+ throw new Error(reason);
+ }
+ if (await isBinaryVobSubSubtitle(file)) {
+ const reason =
+ "VobSub bitmap subtitles aren't supported. Convert the .sub file to SRT or VTT before attaching it.";
+ toast.error(reason);
+ throw new Error(reason);
+ }
+ if (await isBinaryTrackerModule(file)) {
+ const reason =
+ "Tracker .mod audio files aren't supported as text attachments.";
+ toast.error(reason);
+ throw new Error(reason);
+ }
+ if (await isCompiledFortranModule(file)) {
+ const reason =
+ "Compiled Fortran .mod modules aren't supported as text attachments.";
+ toast.error(reason);
+ throw new Error(reason);
+ }
+ if (await isBinaryOfficeTemplate(file)) {
+ const reason =
+ "Legacy Word and PowerPoint templates aren't supported as text attachments.";
+ toast.error(reason);
+ throw new Error(reason);
+ }
+ // Decodes here so an unreadable encoding is reported while attaching rather
+ // than at send, where the composer has no room to explain it.
+ try {
+ await readTextAttachmentOnce(file);
+ } catch (error) {
+ if (error instanceof UndecodableTextError) {
+ toast.error(error.message);
+ }
+ throw error;
+ }
return {
id: crypto.randomUUID(),
type: "document",
@@ -381,7 +468,7 @@ class TextAttachmentAdapter implements AttachmentAdapter {
}
async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
- const text = await attachment.file.text();
+ const text = await readTextAttachmentOnce(attachment.file);
return {
id: attachment.id,
type: "document",
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 314ba70463e..6f5eff8b0b5 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -44,12 +44,13 @@ import {
} from "@/features/settings";
import { useVoiceSettingsStore } from "@/features/settings/stores/voice-settings-store";
import {
- AUDIO_ACCEPT,
+ AUDIO_PICKER_ACCEPT,
+ isAudioAttachmentFile,
fileToBase64,
getAudioSizeError,
} from "@/lib/audio-utils";
import { isTauri } from "@/lib/api-base";
-import { isVideoFile } from "@/lib/video-utils";
+import { classifiedAttachmentFiles, isVideoFile } from "@/lib/video-utils";
import { isDownloadCancelled } from "@/lib/native-files";
import { isMultimodalResponse } from "./types/api";
import { getImageInputUnavailableReason } from "./utils/image-input-support";
@@ -953,8 +954,11 @@ export function SharedComposer({
}, [text]);
const addFiles = useCallback(
- (files: FileList | readonly File[] | null) => {
- if (!files?.length) return;
+ async (input: FileList | readonly File[] | null) => {
+ if (!input?.length) return;
+ // Compare takes audio, so an audio-only 3GP must not be read off its
+ // extension as a clip and refused with the video message.
+ const files = await classifiedAttachmentFiles(input);
const next: PendingImage[] = [];
let droppedImageForUnavailable = false;
let audioSizeError: string | null = null;
@@ -963,7 +967,7 @@ export function SharedComposer({
const file = files[i];
if (!file) continue;
// Handle audio files
- if (file.type.match(/^audio\//i)) {
+ if (isAudioAttachmentFile(file)) {
const sizeError = getAudioSizeError(file.size);
if (sizeError) {
audioSizeError ??= sizeError;
@@ -1010,16 +1014,19 @@ export function SharedComposer({
(event: ClipboardEvent<HTMLTextAreaElement>) => {
pasteClipboardFiles(
event,
- async (files) => {
+ async (pasted) => {
+ // Classify before the check, so a pasted audio-only 3GP is not read
+ // as unsupported on its extension alone.
+ const files = await classifiedAttachmentFiles(pasted);
// Let addFiles report audio size errors.
const supported = files.some(
(file) =>
- file.type.match(/^audio\//i) ||
+ isAudioAttachmentFile(file) ||
(file.type.match(/^image\/(jpeg|png|webp|gif)$/i) &&
file.size <= MAX_IMAGE_SIZE),
);
if (!supported) throw new Error("Unsupported compare attachment");
- addFiles(files);
+ await addFiles(files);
},
() =>
toast.error("Could not paste files.", {
@@ -2197,7 +2204,7 @@ export function SharedComposer({
if (isTauri || isPortaledDrop(e)) return;
e.preventDefault();
setDragging(false);
- addFiles(e.dataTransfer.files);
+ void addFiles(e.dataTransfer.files);
}}
>
<PromptStorageDialog
@@ -2332,17 +2339,17 @@ export function SharedComposer({
multiple
className="hidden"
onChange={(e) => {
- addFiles(e.target.files);
+ void addFiles(e.target.files);
e.target.value = "";
}}
/>
<input
ref={audioInputRef}
type="file"
- accept={AUDIO_ACCEPT}
+ accept={AUDIO_PICKER_ACCEPT}
className="hidden"
onChange={(e) => {
- addFiles(e.target.files);
+ void addFiles(e.target.files);
e.target.value = "";
}}
/>
diff --git a/studio/frontend/src/features/chat/text-attachment-accept.ts b/studio/frontend/src/features/chat/text-attachment-accept.ts
index 1cfc678f91b..fc695780341 100644
--- a/studio/frontend/src/features/chat/text-attachment-accept.ts
+++ b/studio/frontend/src/features/chat/text-attachment-accept.ts
@@ -1,8 +1,17 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-/** Extensions `TextAttachmentAdapter` reads inline; shared with the drop path. */
+/**
+ * Extensions `TextAttachmentAdapter` reads inline; shared with the drop path.
+ *
+ * Read as UTF-8 (or BOM-marked UTF-16) and sent as text, so anything an editor
+ * opens belongs here.
+ * Extensions another adapter claims stay off: the composite adapter takes the
+ * first match, so .html, .pdf, .ods/.odt and the media containers are theirs.
+ * .ts and .mts are TypeScript here, so video cannot take them for MPEG-TS.
+ */
export const TEXT_ATTACHMENT_EXTENSIONS = [
+ // Prose and documentation
".txt",
".text",
".log",
@@ -10,11 +19,38 @@ export const TEXT_ATTACHMENT_EXTENSIONS = [
".markdown",
".mdx",
".rst",
+ ".adoc",
+ ".asciidoc",
+ ".org",
+ ".textile",
+ ".wiki",
+ ".tex",
+ ".latex",
+ ".sty",
+ ".cls",
+ ".bib",
+ ".rmd",
+ ".qmd",
+ // Subtitles and captions
+ ".srt",
+ ".vtt",
+ ".sbv",
+ ".ass",
+ ".ssa",
+ ".sub",
+ ".lrc",
+ // Structured data
".csv",
".tsv",
+ ".psv",
".json",
".jsonl",
".ndjson",
+ ".jsonc",
+ ".json5",
+ ".geojson",
+ ".har",
+ ".avsc",
".xml",
".yaml",
".yml",
@@ -22,72 +58,1340 @@ export const TEXT_ATTACHMENT_EXTENSIONS = [
".ini",
".cfg",
".conf",
+ ".cnf",
".env",
".properties",
+ ".plist",
+ ".edn",
+ ".ron",
+ ".cue",
+ ".lock",
+ ".mod",
+ ".sum",
+ ".reg",
+ ".desktop",
+ ".service",
+ // Localisation and interchange
+ ".po",
+ ".pot",
+ ".strings",
+ ".resx",
+ ".xliff",
+ ".xlf",
+ ".ics",
+ ".vcf",
+ ".eml",
+ ".mbox",
+ ".m3u8",
+ ".pls",
+ // Stylesheets and web templates
".css",
".scss",
".sass",
".less",
+ ".styl",
".svg",
+ ".vue",
+ ".svelte",
+ ".astro",
+ ".pug",
+ ".jade",
+ ".haml",
+ ".slim",
+ ".ejs",
+ ".erb",
+ ".hbs",
+ ".handlebars",
+ ".mustache",
+ ".njk",
+ ".jinja",
+ ".jinja2",
+ ".j2",
+ ".twig",
+ ".liquid",
+ ".cshtml",
+ ".razor",
+ ".aspx",
+ ".jsp",
+ ".tpl",
+ ".qml",
+ // JavaScript and TypeScript
".js",
".jsx",
".mjs",
".cjs",
".ts",
".tsx",
+ ".mts",
+ ".cts",
+ // Python
".py",
".pyi",
+ ".pyx",
+ ".pxd",
".ipynb",
- ".rb",
- ".php",
- ".go",
- ".rs",
+ // JVM
".java",
".kt",
".kts",
".scala",
- ".swift",
+ ".groovy",
+ ".gradle",
+ ".sbt",
+ ".clj",
+ ".cljs",
+ ".cljc",
+ // Systems languages
".c",
".h",
".cc",
".cpp",
".hpp",
".cxx",
+ ".hxx",
+ ".hh",
+ ".ipp",
+ ".inl",
+ ".cu",
+ ".cuh",
+ ".rs",
+ ".go",
+ ".zig",
+ ".odin",
+ ".nim",
+ ".nims",
+ ".nimble",
+ ".cr",
+ ".d",
+ ".v",
+ ".sv",
+ ".svh",
+ ".vhd",
+ ".vhdl",
+ ".asm",
+ ".s",
+ // .NET
".cs",
+ ".vb",
+ ".vbs",
+ ".fs",
+ ".fsi",
+ ".fsx",
+ ".csproj",
+ ".vbproj",
+ ".fsproj",
+ ".sln",
+ ".props",
+ ".targets",
+ // Apple platforms
".m",
".mm",
+ ".swift",
+ ".applescript",
+ ".metal",
+ // Everything else with a compiler or interpreter
+ ".rb",
+ ".rake",
+ ".gemspec",
+ ".podspec",
+ ".php",
+ ".pl",
+ ".pm",
+ ".r",
+ ".jl",
+ ".lua",
+ ".tcl",
+ ".dart",
+ ".hx",
+ ".hs",
+ ".lhs",
+ ".ml",
+ ".mli",
+ ".ex",
+ ".exs",
+ ".erl",
+ ".hrl",
+ ".rkt",
+ ".scm",
+ ".ss",
+ ".lisp",
+ ".lsp",
+ ".cl",
+ ".el",
+ ".pas",
+ ".pp",
+ ".ada",
+ ".adb",
+ ".ads",
+ ".cob",
+ ".cbl",
+ ".f",
+ ".for",
+ ".f90",
+ ".f95",
+ ".f03",
+ ".sas",
+ ".awk",
+ ".sed",
+ ".m4",
+ ".sol",
+ ".move",
+ ".cairo",
+ ".mojo",
+ ".gd",
+ // Shells
".sh",
".bash",
".zsh",
".fish",
+ ".ksh",
+ ".csh",
+ ".tcsh",
+ ".nu",
".ps1",
+ ".psm1",
+ ".psd1",
".bat",
- ".lua",
- ".pl",
- ".pm",
- ".r",
- ".jl",
- ".dart",
- ".vue",
- ".svelte",
- ".astro",
+ ".cmd",
+ // Queries and schemas
".sql",
+ ".psql",
+ ".plsql",
+ ".hql",
+ ".cql",
".graphql",
".gql",
".proto",
+ ".thrift",
+ ".capnp",
+ ".prisma",
+ // Infrastructure and build
".tf",
".tfvars",
- ".gradle",
+ ".tfstate",
+ ".hcl",
+ ".nix",
+ ".dhall",
+ ".bicep",
".dockerfile",
+ ".containerfile",
".makefile",
+ ".mk",
+ ".mak",
".cmake",
+ ".ninja",
+ ".bzl",
+ ".bazel",
+ ".star",
+ ".starlark",
+ ".gn",
+ ".gni",
+ ".pro",
+ ".pri",
+ ".cabal",
+ ".opam",
+ // Shaders
+ ".glsl",
+ ".frag",
+ ".vert",
+ ".geom",
+ ".comp",
+ ".hlsl",
+ ".wgsl",
+ ".shader",
+ // Diagrams, specs and request files
+ ".mmd",
+ ".mermaid",
+ ".puml",
+ ".plantuml",
+ ".dot",
+ ".gv",
+ ".feature",
+ ".robot",
+ ".http",
+ ".rest",
+ // Diffs
".diff",
".patch",
];
+/** Matches MAX_NATIVE_TEXT_BYTES in native_intents.rs, so a file dropped from
+ * the desktop shell and one picked in the browser accept the same sizes. An
+ * .mbox can run to gigabytes, and reading one decodes it twice over in memory. */
+export const MAX_TEXT_ATTACHMENT_BYTES = 20 * 1024 * 1024;
+
+/** Conventional extensionless names matched through their dotted adapter tokens.
+ * assistant-ui reads "Dockerfile" as the extension ".dockerfile", so the picker
+ * already claims these; the drop paths have to agree. */
+export const TEXT_ATTACHMENT_BASENAMES = [
+ "containerfile",
+ "dockerfile",
+ "makefile",
+] as const;
+const PATH_SEPARATOR_RE = /[\\/]/;
+
+/** Whether a path or filename belongs to the inline UTF-8 text adapter. */
+export function isTextAttachmentName(path: string): boolean {
+ const segments = path.split(PATH_SEPARATOR_RE);
+ const name = (segments[segments.length - 1] || path).toLowerCase();
+ if ((TEXT_ATTACHMENT_BASENAMES as readonly string[]).includes(name)) {
+ return true;
+ }
+ const dot = name.lastIndexOf(".");
+ return dot > 0 && TEXT_ATTACHMENT_EXTENSIONS.includes(name.slice(dot));
+}
+
+/**
+ * HTML's `accept` attribute can express MIME types and extensions, not an exact
+ * extensionless filename. Omit that picker hint when this adapter is present;
+ * the attachment adapters still validate every selected file.
+ */
+export function pickerAcceptForTextBasenames(accept: string): string {
+ if (accept === "*") {
+ return accept;
+ }
+ const tokens = accept.split(",").map((token) => token.trim().toLowerCase());
+ return TEXT_ATTACHMENT_BASENAMES.some((name) => tokens.includes(`.${name}`))
+ ? "*"
+ : accept;
+}
+
+/** Binary Apple property lists are not text despite using `.plist` or `.strings`. */
+export async function isBinaryPropertyList(file: File): Promise<boolean> {
+ if (!/\.(?:plist|strings)$/i.test(file.name)) {
+ return false;
+ }
+ const header = new Uint8Array(await file.slice(0, 8).arrayBuffer());
+ return header.length === 8 && String.fromCharCode(...header) === "bplist00";
+}
+
+/** VobSub `.sub` files are MPEG program streams containing bitmap subtitles. */
+export async function isBinaryVobSubSubtitle(file: File): Promise<boolean> {
+ if (!file.name.toLowerCase().endsWith(".sub")) {
+ return false;
+ }
+ const header = new Uint8Array(await file.slice(0, 4).arrayBuffer());
+ return (
+ header.length === 4 &&
+ header[0] === 0x00 &&
+ header[1] === 0x00 &&
+ header[2] === 0x01 &&
+ header[3] === 0xba
+ );
+}
+
+const TRACKER_MOD_MAGICS = new Set([
+ "M.K.",
+ "M!K!",
+ // ProTracker's other 4-channel marker. file(1) lists it at offset 1080
+ // alongside M.K., and the 31-sample layout puts byte 470 inside a sample name
+ // rather than the order table, so the Soundtracker fallback does not catch
+ // one either: without this the module is read as UTF-8 text.
+ "!PM!",
+ "PATT",
+ "NSMS",
+ "LARD",
+ "M&K!",
+ "FEST",
+ "N.T.",
+ "OKTA",
+ "OCTA",
+ "CD81",
+ "CD61",
+ "FLT4",
+ "FLT8",
+ "EXO4",
+ "EXO8",
+ ".M.K",
+ "WARD",
+ "M\0\0\0",
+ "8\0\0\0",
+]);
+const TRACKER_SINGLE_CHANNEL_MAGIC_RE = /^[1-9]CHN$/;
+const TRACKER_DOUBLE_CHANNEL_MAGIC_RE = /^[1-9][0-9](?:CH|CN)$/;
+const TRACKER_TAKE_MAGIC_RE = /^TDZ[1-9]$/;
+const TRACKER_DIGITAL_MAGIC_RE = /^FA0[4-8]$/;
+
+const SOUNDTRACKER_HEADER_BYTES = 600;
+const SOUNDTRACKER_PATTERN_BYTES = 1024;
+const SOUNDTRACKER_MAX_PATTERN_ERRORS = 22;
+const SOUNDTRACKER_PERIODS = new Set([
+ 856, 808, 762, 720, 678, 640, 604, 570, 538, 508, 480, 453, 428, 404,
+ 381, 360, 339, 320, 302, 285, 269, 254, 240, 226, 214, 202, 190, 180,
+ 170, 160, 151, 143, 135, 127, 120, 113, 763, 679, 641, 571, 539, 509,
+ 429, 340, 321, 300, 286, 270, 227, 191, 162,
+]);
+
+function bigEndianWord(bytes: Uint8Array, offset: number): number {
+ return bytes[offset] * 256 + bytes[offset + 1];
+}
+
+function soundtrackerSampleBytes(header: Uint8Array): number | null {
+ let sampleBytes = 0;
+ for (let sample = 0; sample < 15; sample += 1) {
+ const offset = 20 + sample * 30;
+ const lengthWords = bigEndianWord(header, offset + 22);
+ const finetune = header[offset + 24];
+ const volume = header[offset + 25];
+ const loopStart = bigEndianWord(header, offset + 26);
+ const loopLength = bigEndianWord(header, offset + 28);
+ if (
+ volume > 0x40 ||
+ (finetune & 0xf0) !== 0 ||
+ lengthWords > 0x8000 ||
+ loopLength > 0x8000 ||
+ (loopStart >>> 1) > lengthWords ||
+ (lengthWords > 0 && (loopStart >>> 1) === lengthWords) ||
+ (lengthWords === 0 && loopStart > 0)
+ ) {
+ return null;
+ }
+ sampleBytes += lengthWords * 2;
+ }
+ return sampleBytes >= 8 ? sampleBytes : null;
+}
+
+function soundtrackerPatternCounts(
+ header: Uint8Array,
+): { all: number; used: number } | null {
+ const songLength = header[470];
+ if (songLength === 0 || songLength > 128) {
+ return null;
+ }
+ let maxPattern = 0;
+ let maxUsedPattern = 0;
+ for (let index = 0; index < 128; index += 1) {
+ const pattern = header[472 + index];
+ if (pattern > 0x7f) {
+ return null;
+ }
+ maxPattern = Math.max(maxPattern, pattern);
+ if (index < songLength) {
+ maxUsedPattern = Math.max(maxUsedPattern, pattern);
+ }
+ }
+ return { all: maxPattern + 1, used: maxUsedPattern + 1 };
+}
+
+function soundtrackerPatternEnd(
+ header: Uint8Array,
+ fileSize: number,
+): number | null {
+ if (header.length < SOUNDTRACKER_HEADER_BYTES) {
+ return null;
+ }
+
+ const sampleBytes = soundtrackerSampleBytes(header);
+ const patternCounts = soundtrackerPatternCounts(header);
+ if (sampleBytes === null || patternCounts === null) {
+ return null;
+ }
+
+ let patternCount = patternCounts.all;
+ const usedPatternCount = patternCounts.used;
+ let expectedSize =
+ SOUNDTRACKER_HEADER_BYTES +
+ patternCount * SOUNDTRACKER_PATTERN_BYTES +
+ sampleBytes;
+ const usedExpectedSize =
+ SOUNDTRACKER_HEADER_BYTES +
+ usedPatternCount * SOUNDTRACKER_PATTERN_BYTES +
+ sampleBytes;
+ // Some old files leave junk orders past the declared song length.
+ if (fileSize < expectedSize && fileSize === usedExpectedSize) {
+ patternCount = usedPatternCount;
+ expectedSize = usedExpectedSize;
+ }
+ // Known Soundtracker files can have truncated trailing sample data, but the
+ // complete pattern area must still be present.
+ const patternEnd =
+ SOUNDTRACKER_HEADER_BYTES + patternCount * SOUNDTRACKER_PATTERN_BYTES;
+ return fileSize >= Math.floor((expectedSize * 93) / 100) &&
+ fileSize >= patternEnd
+ ? patternEnd
+ : null;
+}
+
+function hasSoundtrackerPatternData(
+ bytes: Uint8Array,
+ patternEnd: number,
+): boolean {
+ if (bytes.length < patternEnd) {
+ return false;
+ }
+ let errors = 0;
+ for (
+ let offset = SOUNDTRACKER_HEADER_BYTES;
+ offset < patternEnd;
+ offset += 4
+ ) {
+ const sample = (bytes[offset] & 0xf0) | (bytes[offset + 2] >>> 4);
+ const period = ((bytes[offset] & 0x0f) << 8) | bytes[offset + 1];
+ if (sample > 15) {
+ errors += 1;
+ if (errors > SOUNDTRACKER_MAX_PATTERN_ERRORS) {
+ return false;
+ }
+ }
+ if (period !== 0 && !SOUNDTRACKER_PERIODS.has(period)) {
+ errors += 1;
+ if (errors > SOUNDTRACKER_MAX_PATTERN_ERRORS) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+/** gfortran writes a compiled `.mod` module as gzip; text `go.mod` never is. */
+export async function isCompiledFortranModule(file: File): Promise<boolean> {
+ if (!file.name.toLowerCase().endsWith(".mod")) {
+ return false;
+ }
+ const header = new Uint8Array(await file.slice(0, 2).arrayBuffer());
+ return header.length === 2 && header[0] === 0x1f && header[1] === 0x8b;
+}
+
+/** Detect marker-bearing MODs and earlier 15-sample Soundtracker modules. */
+export async function isBinaryTrackerModule(file: File): Promise<boolean> {
+ if (!file.name.toLowerCase().endsWith(".mod")) {
+ return false;
+ }
+ const prefix = new Uint8Array(await file.slice(0, 1084).arrayBuffer());
+ if (prefix.length >= 1084) {
+ const magic = String.fromCharCode(...prefix.subarray(1080, 1084));
+ if (
+ TRACKER_MOD_MAGICS.has(magic) ||
+ TRACKER_SINGLE_CHANNEL_MAGIC_RE.test(magic) ||
+ TRACKER_DOUBLE_CHANNEL_MAGIC_RE.test(magic) ||
+ TRACKER_TAKE_MAGIC_RE.test(magic) ||
+ TRACKER_DIGITAL_MAGIC_RE.test(magic)
+ ) {
+ return true;
+ }
+ }
+
+ const patternEnd = soundtrackerPatternEnd(prefix, file.size);
+ if (patternEnd === null) {
+ return false;
+ }
+ const patterns = new Uint8Array(
+ await file.slice(0, patternEnd).arrayBuffer(),
+ );
+ return hasSoundtrackerPatternData(patterns, patternEnd);
+}
+
+const GETTEXT_HEADER_SCAN_BYTES = 64 * 1024;
+const GETTEXT_CHARSET_ALIASES: Record<string, string> = {
+ CP874: "windows-874",
+ CP932: "shift_jis",
+ CP949: "euc-kr",
+ CP950: "big5",
+};
+const GETTEXT_HEADER_ENTRY_RE =
+ /(?:^|\r?\n)msgid[ \t]+""[ \t]*\r?\nmsgstr[ \t]+("(?:[^"\\]|\\.)*"(?:[ \t]*\r?\n[ \t]*"(?:[^"\\]|\\.)*")*)/;
+const GETTEXT_CHARSET_RE =
+ /Content-Type:[^"\r\n]*?charset[ \t]*=[ \t]*([A-Za-z0-9._-]+)/i;
+
+/**
+ * The charset a gettext catalog's header entry declares.
+ *
+ * The entry is the first one in the file by convention, so the prefix settles
+ * it without decoding a catalog that can run to megabytes. A cutoff can still
+ * miss it, though: a project with more than 64 KiB of translator comments above
+ * the entry pushes it past the prefix, and the catalog was then refused for
+ * carrying exactly the bytes its header describes. So when the prefix holds no
+ * entry, read the rest, which the attachment cap already bounds.
+ */
+function declaredGettextCharset(
+ bytes: Uint8Array,
+ fileName: string,
+): string | null {
+ if (!/\.(?:po|pot)$/i.test(fileName)) {
+ return null;
+ }
+ const decoder = new TextDecoder("windows-1252");
+ const cut = bytes.length > GETTEXT_HEADER_SCAN_BYTES;
+ const fromPrefix = gettextHeaderCharset(
+ decoder.decode(bytes.subarray(0, GETTEXT_HEADER_SCAN_BYTES)),
+ cut,
+ );
+ if (fromPrefix || !cut) {
+ return fromPrefix;
+ }
+ // A cutoff can also split the entry itself, leaving a match that holds the
+ // first continuation lines but not the Content-Type one, so retry on the
+ // charset rather than on the entry.
+ return gettextHeaderCharset(decoder.decode(bytes));
+}
+
+// The C escapes gettext writes. Anything else stands for itself, which is what
+// \" and \\ need, and leaves an unknown one as the character it escaped.
+const GETTEXT_ESCAPES: Record<string, string> = {
+ a: "\x07",
+ b: "\b",
+ f: "\f",
+ n: "\n",
+ r: "\r",
+ t: "\t",
+ v: "\v",
+};
+
+/**
+ * A gettext string's value: its adjacent literals joined, escapes resolved.
+ *
+ * PO wraps a long string across quoted pieces and concatenates them, so the
+ * header is not the source text. Matching the raw entry instead read a charset
+ * split across two pieces (`charset=windows-` then `1252\n`) as the truncated
+ * label, and a catalog was refused for declaring a charset it does not.
+ */
+function gettextStringValue(raw: string): string {
+ let out = "";
+ let inside = false;
+ for (let index = 0; index < raw.length; index += 1) {
+ const character = raw[index];
+ if (!inside) {
+ if (character === '"') inside = true;
+ continue;
+ }
+ if (character === "\\" && index + 1 < raw.length) {
+ const escaped = raw[index + 1];
+ out += GETTEXT_ESCAPES[escaped] ?? escaped;
+ index += 1;
+ } else if (character === '"') {
+ inside = false;
+ } else {
+ out += character;
+ }
+ }
+ return out;
+}
+
+// A line that opens with anything but another quoted piece, which is where a
+// header entry ends.
+const GETTEXT_ENTRY_ENDED_RE = /\r?\n[ \t]*[^ \t"\r\n]/;
+
+/**
+ * @param truncated Whether `text` is a prefix of the file. An entry that runs
+ * to the cut may continue past it, and half a `charset=windows-1252` reads as
+ * the label `windows-`, so the caller is told to look at the whole file rather
+ * than given an answer drawn from half a value.
+ */
+function gettextHeaderCharset(text: string, truncated = false): string | null {
+ const match = text.match(GETTEXT_HEADER_ENTRY_RE);
+ if (!match) {
+ return null;
+ }
+ const after = text.slice((match.index ?? 0) + match[0].length);
+ if (truncated && !GETTEXT_ENTRY_ENDED_RE.test(after)) {
+ return null;
+ }
+ const charset = gettextStringValue(match[1]).match(GETTEXT_CHARSET_RE)?.[1];
+ // A .pot template ships the literal placeholder rather than a charset.
+ return charset && charset.toUpperCase() !== "CHARSET" ? charset : null;
+}
+
+// Header block only: a body part can declare its own, but the message-level
+// charset is what an 8-bit single-part mail is written in.
+// Every Content-Type in the file, not just the first: a multipart message keeps
+// the charset on its parts, and an mbox holds one per message, which can sit
+// anywhere in the archive. Folded continuation lines start with space or tab.
+const EMAIL_CONTENT_TYPE_RE =
+ /(?:^|\r?\n)Content-Type:((?:[^\r\n]*)(?:\r?\n[ \t][^\r\n]*)*)/gi;
+const CHARSET_LABEL_RE = /^[A-Za-z0-9._-]+/;
+
+/** A header value's parameters, media type excluded, split outside quotes. */
+function headerParameters(value: string): string[] {
+ const parameters: string[] = [];
+ let start = 0;
+ let quoted = false;
+ for (let index = 0; index < value.length; index += 1) {
+ const character = value[index];
+ if (character === "\\" && quoted) {
+ index += 1;
+ } else if (character === '"') {
+ quoted = !quoted;
+ } else if (character === ";" && !quoted) {
+ parameters.push(value.slice(start, index));
+ start = index + 1;
+ }
+ }
+ parameters.push(value.slice(start));
+ return parameters.slice(1);
+}
+
+/**
+ * A header value's parameter by name, and nothing that merely looks like one.
+ *
+ * An unanchored search took the first match anywhere in the value, so
+ * `text/plain; name="charset=windows-1251.txt"; charset=windows-1252` read the
+ * filename and stopped, and a `filename="report; boundary=fake"` on any header
+ * at all registered a multipart delimiter that does not exist.
+ */
+function headerParameter(value: string, name: string): string | undefined {
+ for (const parameter of headerParameters(value)) {
+ const equals = parameter.indexOf("=");
+ if (equals === -1) continue;
+ if (parameter.slice(0, equals).trim().toLowerCase() !== name) continue;
+ const raw = parameter.slice(equals + 1).trim();
+ if (!raw.startsWith('"')) return raw || undefined;
+ return unquoteHeaderValue(raw) || undefined;
+ }
+ return undefined;
+}
+
+/**
+ * A quoted-string parameter value, escapes resolved.
+ *
+ * `\X` stands for `X`, so a value carrying a quote reads to its real closing
+ * one rather than stopping at the escaped one, and the text it yields is what a
+ * delimiter line actually holds. The splitter above already honours escapes;
+ * this is the other half of that.
+ */
+function unquoteHeaderValue(raw: string): string {
+ let out = "";
+ for (let index = 1; index < raw.length; index += 1) {
+ const character = raw[index];
+ if (character === "\\" && index + 1 < raw.length) {
+ out += raw[index + 1];
+ index += 1;
+ } else if (character === '"') {
+ break;
+ } else {
+ out += character;
+ }
+ }
+ return out;
+}
+
+/** The charset a header declares, restricted to the shape of a label. */
+function headerCharsetParameter(value: string): string | undefined {
+ return headerParameter(value, "charset")?.match(CHARSET_LABEL_RE)?.[0];
+}
+
+// vCard 2.1 puts the encoding on the property: `FN;CHARSET=windows-1252:...`.
+const VCARD_CHARSET_RE = /;[ \t]*CHARSET[ \t]*=[ \t]*"?([A-Za-z0-9._-]+)"?/gi;
+// Only a multipart type carries a delimiter for its own parts.
+const MULTIPART_TYPE_RE = /^[ \t]*multipart\//i;
+// A header continues on the next line when that line begins with whitespace.
+const HEADER_FOLD_RE = /\r?\n[ \t]+/g;
+
+// A header block runs from the start of the file, an mbox "From " separator, or
+// a MIME boundary, to the first blank line. Body text can hold a line that reads
+// like a header, and treating that as a declaration refused valid files.
+/**
+ * The header regions of a message or archive, body text excluded.
+ *
+ * One forward pass over the lines, slicing only the header regions themselves.
+ * Restarting a search from each candidate boundary instead is quadratic, and a
+ * body of diff hunks is all candidate boundaries: 1 MB of them took 14 seconds.
+ *
+ * Only a delimiter a header actually declared reopens the headers. Any line
+ * starting with `--` reopened them before, so a signature marker or a quoted
+ * diff put the body back into header mode, and the next body line shaped like
+ * `Content-Type: ... charset=...` became a second declaration that refused the
+ * file. A closing delimiter, `--boundary--`, ends its part instead of starting
+ * one.
+ *
+ * @param mbox Whether the file is an archive of messages. A `From ` line only
+ * separates messages there, and an archive escapes the one a body starts with.
+ * A single `.eml` has no separator at all, so an ordinary sentence opening with
+ * "From " is body text and must not reopen the headers.
+ */
+function emailHeaderBlocks(text: string, mbox: boolean): string[] {
+ const blocks: string[] = [];
+ const boundaries = new Set<string>();
+ const closeBlock = (block: string) => {
+ // Unfolded first: a header may wrap anywhere folding whitespace is allowed,
+ // and `multipart/mixed;\r\n boundary="part"` is a common wrap. Reading the
+ // raw block missed that boundary, so the part headers below it were never
+ // scanned and the charset they declare went with them.
+ const unfolded = block.replace(HEADER_FOLD_RE, " ");
+ blocks.push(unfolded);
+ for (const header of unfolded.matchAll(EMAIL_CONTENT_TYPE_RE)) {
+ const value = header[1] ?? "";
+ // A boundary means something only on the header that owns the parts. Any
+ // header's quoted filename could otherwise register a delimiter, and a
+ // body line repeating it then reopened the headers.
+ if (!MULTIPART_TYPE_RE.test(value)) continue;
+ const boundary = headerParameter(value, "boundary");
+ if (boundary) boundaries.add(boundary);
+ }
+ };
+ let position = 0;
+ let blockStart = 0;
+ let inHeader = true;
+ while (position <= text.length) {
+ let lineBreak = text.indexOf("\n", position);
+ if (lineBreak === -1) lineBreak = text.length;
+ const contentEnd =
+ lineBreak > position && text[lineBreak - 1] === "\r"
+ ? lineBreak - 1
+ : lineBreak;
+ const isBlank = contentEnd === position;
+ if (inHeader) {
+ if (isBlank) {
+ closeBlock(text.slice(blockStart, position));
+ inHeader = false;
+ }
+ } else if (!isBlank) {
+ // An mbox separator, or a delimiter one of the headers above declared.
+ let resumes = mbox && text.startsWith("From ", position);
+ if (resumes) {
+ // A boundary belongs to the message that named it. Carrying one into
+ // the next message lets a body line that happens to repeat it reopen
+ // the headers there.
+ boundaries.clear();
+ } else if (boundaries.size > 0 && text.startsWith("--", position)) {
+ // Trailing whitespace is allowed on a delimiter line and is not part of
+ // the boundary token. A closing delimiter carries the extra "--" into
+ // the token, so it does not match and does not reopen the headers.
+ const token = text
+ .slice(position + 2, contentEnd)
+ .replace(/[ \t]+$/, "");
+ resumes = boundaries.has(token);
+ if (!resumes && token.endsWith("--")) {
+ // `--part--` ends that multipart for good, so the delimiter stops
+ // being one. A nested boundary kept past its close let a later
+ // sibling part repeating the line reopen the headers.
+ boundaries.delete(token.slice(0, -2));
+ }
+ }
+ if (resumes) {
+ inHeader = true;
+ blockStart = lineBreak + 1;
+ }
+ }
+ if (lineBreak === text.length) break;
+ position = lineBreak + 1;
+ }
+ if (inHeader && blockStart < text.length) {
+ closeBlock(text.slice(blockStart));
+ }
+ return blocks;
+}
+
+/**
+ * The property-parameter sections of a vCard, values excluded.
+ *
+ * `FN;CHARSET=windows-1252:name` declares a charset; `NOTE:see \;CHARSET=x`
+ * does not, because that text is the value. The section therefore ends at the
+ * property's value delimiter: the first colon outside a quoted parameter value.
+ * Folded lines, which begin with a space or tab, continue the line above.
+ */
+function vCardParameterSections(text: string): string[] {
+ const sections: string[] = [];
+ let section = "";
+ let valueReached = true;
+ let quoted = false;
+ let position = 0;
+ while (position <= text.length) {
+ let lineBreak = text.indexOf("\n", position);
+ if (lineBreak === -1) lineBreak = text.length;
+ const contentEnd =
+ lineBreak > position && text[lineBreak - 1] === "\r"
+ ? lineBreak - 1
+ : lineBreak;
+ const folded = text[position] === " " || text[position] === "\t";
+ let start = position;
+ if (folded) {
+ start = position + 1;
+ } else {
+ if (section) sections.push(section);
+ section = "";
+ valueReached = false;
+ quoted = false;
+ }
+ if (!valueReached) {
+ let cut = contentEnd;
+ for (let index = start; index < contentEnd; index += 1) {
+ const character = text[index];
+ if (character === '"') {
+ quoted = !quoted;
+ } else if (character === ":" && !quoted) {
+ cut = index;
+ valueReached = true;
+ break;
+ }
+ }
+ section += text.slice(start, cut);
+ }
+ if (lineBreak === text.length) break;
+ position = lineBreak + 1;
+ }
+ if (section) sections.push(section);
+ return sections;
+}
+
+// An XML prolog names the document's encoding and must be the first thing in it.
+// Read from the bytes rather than the extension, so every XML dialect here
+// (.resx, .xliff, .svg, a text .plist, the project files) is covered at once.
+// The whitespace after the target is what tells a declaration from a processing
+// instruction: `xml-stylesheet` and `xml-model` open with the same five bytes,
+// and neither states what the document is written in.
+const XML_PROLOG_ENCODING_RE =
+ /^<\?xml[ \t\r\n][^>]*?[ \t\r\n]encoding[ \t]*=[ \t]*["\']([A-Za-z0-9._-]+)["\']/i;
+// XML's own S production. `\s` would also take a form feed or a no-break space,
+// which the grammar does not, so a PI could pass on one of those instead.
+const XML_WHITESPACE_BYTES = new Set([0x20, 0x09, 0x0d, 0x0a]);
+
+/**
+ * The encoding an XML declaration names, if the document opens with one.
+ *
+ * The declaration has to be the first thing in the document, so five bytes
+ * settle whether there is one to read; everything else costs nothing. It then
+ * runs to its own ">", and its grammar puts no bound on the whitespace between
+ * the parts, so a fixed prefix could cut `encoding` off and leave a document
+ * that states its encoding refused for holding it.
+ */
+function declaredXmlEncoding(bytes: Uint8Array): string | null {
+ const decoder = new TextDecoder("windows-1252");
+ if (decoder.decode(bytes.subarray(0, 5)).toLowerCase() !== "<?xml") {
+ return null;
+ }
+ // A sixth byte of whitespace, or this is a PI target that merely begins the
+ // same way. Reading one of those as the declaration decoded a UTF-8 document
+ // as a code page, and mojibake is what it returned rather than an error.
+ if (bytes.length < 6 || !XML_WHITESPACE_BYTES.has(bytes[5])) {
+ return null;
+ }
+ const close = bytes.indexOf(0x3e);
+ if (close === -1) {
+ return null;
+ }
+ const declaration = decoder.decode(bytes.subarray(0, close + 1));
+ return declaration.match(XML_PROLOG_ENCODING_RE)?.[1] ?? null;
+}
+
+/**
+ * The encoding a declared label names, or the label itself when nothing decodes
+ * it.
+ *
+ * `windows-1252`, `CP1252` and `latin1` are three spellings of one encoding, so
+ * comparing the spellings called an archive multi-charset when a single decoder
+ * reads every message in it. TextDecoder already knows the equivalences, and
+ * reports the canonical name it settled on.
+ */
+function canonicalCharset(charset: string): string {
+ const label = GETTEXT_CHARSET_ALIASES[charset.toUpperCase()] ?? charset;
+ try {
+ return new TextDecoder(label).encoding;
+ } catch {
+ return charset.toLowerCase();
+ }
+}
+
+/** Collects distinct charsets in encounter order, one entry per encoding. */
+function charsetCollector(): { found: string[]; add: (c?: string) => void } {
+ const found: string[] = [];
+ const seen = new Set<string>();
+ return {
+ found,
+ add(charset?: string) {
+ if (!charset) return;
+ const canonical = canonicalCharset(charset);
+ if (seen.has(canonical)) return;
+ seen.add(canonical);
+ // The spelling the file used, since that is what an error should name.
+ found.push(charset);
+ },
+ };
+}
+
+// Headers and property parameters are ASCII, so these scans cannot fail on the
+// body's own encoding. They read the whole file rather than a prefix: a
+// declaration further in is the one a cutoff would miss, leaving those messages
+// decoded as the first. The attachment cap already bounds the work.
+
+/** Charsets a `.vcf` declares, one per property parameter. */
+function declaredVCardCharsets(bytes: Uint8Array, fileName: string): string[] {
+ if (!/\.vcf$/i.test(fileName)) {
+ return [];
+ }
+ const text = new TextDecoder("windows-1252").decode(bytes);
+ const { found, add } = charsetCollector();
+ for (const section of vCardParameterSections(text)) {
+ for (const property of section.matchAll(VCARD_CHARSET_RE)) {
+ add(property[1]);
+ }
+ }
+ return found;
+}
+
+/** Charsets an `.eml` or `.mbox` declares, one per message or part header. */
+function declaredEmailCharsets(bytes: Uint8Array, fileName: string): string[] {
+ const isMbox = /\.mbox$/i.test(fileName);
+ if (!isMbox && !/\.eml$/i.test(fileName)) {
+ return [];
+ }
+ const text = new TextDecoder("windows-1252").decode(bytes);
+ const { found, add } = charsetCollector();
+ for (const block of emailHeaderBlocks(text, isMbox)) {
+ for (const header of block.matchAll(EMAIL_CONTENT_TYPE_RE)) {
+ add(headerCharsetParameter(header[1] ?? ""));
+ }
+ }
+ return found;
+}
+
+/** A charset this browser has no decoder for.
+ *
+ * An UndecodableTextError rather than a plain one, because that is the type
+ * the composer recognises: every other refusal in TextAttachmentAdapter.add
+ * toasts before it throws, and this one went out as a bare Error, failed the
+ * `instanceof` there, and left the attachment silently missing. The labels
+ * that land here are not exotic. Six of them are standard: the Encoding
+ * Standard maps iso-2022-kr, hz-gb-2312 and the iso-2022-cn family to
+ * "replacement", which TextDecoder is required to refuse, so a Korean or
+ * Chinese .eml declaring its own charset disappeared with no message at all.
+ */
+function unsupportedCharsetError(
+ fileName: string,
+ charset: string,
+): UndecodableTextError {
+ return new UndecodableTextError(
+ fileName,
+ `It declares charset "${charset}", which this browser has no decoder for.`,
+ );
+}
+
+/** A charset the file declared and then contradicted. */
+function contradictedCharsetError(
+ fileName: string,
+ charset: string,
+): UndecodableTextError {
+ return new UndecodableTextError(
+ fileName,
+ `It declares charset "${charset}" but does not hold valid ${charset} text.`,
+ );
+}
+
+/** Decode under a charset the file itself declared, or say why it could not. */
+function decodeWithCharset(
+ bytes: Uint8Array,
+ charset: string,
+ fileName: string,
+ truncated: boolean,
+): string {
+ const label = GETTEXT_CHARSET_ALIASES[charset.toUpperCase()] ?? charset;
+ let decoder: TextDecoder;
+ try {
+ // Strict, like the default path: a declaration is a claim about the bytes,
+ // and bytes that break it are corrupt rather than readable. Single-byte
+ // charsets map everything, so this only bites on the multibyte ones.
+ decoder = new TextDecoder(label, { fatal: true });
+ } catch (error) {
+ if (error instanceof RangeError) {
+ throw unsupportedCharsetError(fileName, charset);
+ }
+ throw error;
+ }
+ try {
+ return decoder.decode(bytes, { stream: truncated });
+ } catch {
+ throw contradictedCharsetError(fileName, charset);
+ }
+}
+
+/** A strict decoder for a declared charset, or null when it is not supported. */
+function strictDecoder(charset: string): TextDecoder | null {
+ const label = GETTEXT_CHARSET_ALIASES[charset.toUpperCase()] ?? charset;
+ try {
+ return new TextDecoder(label, { fatal: true });
+ } catch {
+ return null;
+ }
+}
+
+// vCard names its properties and parameters in ASCII, so only the value after
+// the delimiter carries the declared charset.
+const ASCII_DECODER_LABEL = "windows-1252";
+
+/**
+ * A vCard decoded one property at a time, or null if it cannot be.
+ *
+ * `CHARSET` is a property parameter, so a 2.1 card may legitimately put
+ * windows-1252 on `FN` and windows-1251 on `NOTE`. Reading the file as one unit
+ * corrupts every property but one, which is why more than one declaration is
+ * refused there; this honours each of them instead.
+ *
+ * Only the value is decoded under the property's own charset. The name and
+ * parameters ahead of the delimiter are ASCII by the grammar, and a folded line,
+ * which begins with a space or tab, continues the property above under the same
+ * charset. A property that declares nothing is UTF-8, strictly: this is not the
+ * place to guess a code page for it.
+ *
+ * Anything it cannot account for comes back as a reading with no text, so the
+ * caller falls back to the whole-file one rather than emitting a partly guessed
+ * one. `failure` is set when a declaration itself is the obstacle, which no
+ * other reading of the file can resolve.
+ *
+ * @param truncated Whether `bytes` is a prefix. Only its last line can end
+ * mid-character, so only that one is read leniently.
+ */
+function decodeVCardPerProperty(
+ bytes: Uint8Array,
+ truncated: boolean,
+ fileName: string,
+): { text: string } | { failure: Error | null } {
+ const ascii = new TextDecoder(ASCII_DECODER_LABEL);
+ const utf8 = new TextDecoder("utf-8", { fatal: true });
+ const decoders = new Map<string, TextDecoder | null>();
+ const decoderFor = (charset: string | null): TextDecoder | null => {
+ if (charset === null) return utf8;
+ const seen = decoders.get(charset);
+ if (seen !== undefined) return seen;
+ const made = strictDecoder(charset);
+ decoders.set(charset, made);
+ return made;
+ };
+
+ const out: string[] = [];
+ let valueDecoder: TextDecoder | null = utf8;
+ // What the property being read declared, so a decode that throws can name it.
+ let valueCharset: string | null = null;
+ let position = 0;
+ while (position < bytes.length) {
+ let lineBreak = bytes.indexOf(0x0a, position);
+ if (lineBreak === -1) lineBreak = bytes.length;
+ const contentEnd =
+ lineBreak > position && bytes[lineBreak - 1] === 0x0d
+ ? lineBreak - 1
+ : lineBreak;
+ const folded = bytes[position] === 0x20 || bytes[position] === 0x09;
+ // Only a prefix read can end mid-character, and only on its final line, so
+ // that one line is decoded leniently and every other stays strict.
+ const cut = truncated && lineBreak === bytes.length;
+ try {
+ if (folded) {
+ // A continuation of the value above, so it keeps that charset.
+ if (!valueDecoder) return { failure: null };
+ out.push(
+ valueDecoder.decode(bytes.subarray(position, contentEnd), {
+ stream: cut,
+ }),
+ );
+ } else {
+ const delimiter = vCardValueDelimiter(bytes, position, contentEnd);
+ if (delimiter === -1) {
+ // Not a property line. Only ASCII can be read without a declaration.
+ out.push(
+ utf8.decode(bytes.subarray(position, contentEnd), { stream: cut }),
+ );
+ valueDecoder = utf8;
+ valueCharset = null;
+ } else {
+ const parameters = ascii.decode(bytes.subarray(position, delimiter));
+ const charsets: string[] = [];
+ for (const match of parameters.matchAll(VCARD_CHARSET_RE)) {
+ if (match[1]) charsets.push(match[1]);
+ }
+ // Two on one property is the container's problem, not this reading's.
+ if (charsets.length > 1) {
+ return {
+ failure: new UndecodableTextError(
+ fileName,
+ `One of its properties declares two charsets (${charsets.join(", ")}).`,
+ ),
+ };
+ }
+ valueCharset = charsets[0] ?? null;
+ valueDecoder = decoderFor(valueCharset);
+ if (!valueDecoder) {
+ return { failure: unsupportedCharsetError(fileName, valueCharset!) };
+ }
+ out.push(parameters, ":");
+ out.push(
+ valueDecoder.decode(bytes.subarray(delimiter + 1, contentEnd), {
+ stream: cut,
+ }),
+ );
+ }
+ }
+ } catch {
+ // A property that broke its own declaration is the declaration's problem.
+ // One with no declaration is just bytes, and the caller may still have a
+ // whole-file charset that reads them.
+ return {
+ failure: valueCharset
+ ? contradictedCharsetError(fileName, valueCharset)
+ : null,
+ };
+ }
+ out.push(contentEnd === lineBreak ? "" : "\r");
+ if (lineBreak === bytes.length) break;
+ out.push("\n");
+ position = lineBreak + 1;
+ }
+ return { text: out.join("") };
+}
+
+/** The first colon outside a quoted parameter value, or -1 if there is none. */
+function vCardValueDelimiter(
+ bytes: Uint8Array,
+ start: number,
+ end: number,
+): number {
+ let quoted = false;
+ for (let index = start; index < end; index += 1) {
+ const byte = bytes[index];
+ if (byte === 0x22) quoted = !quoted;
+ else if (byte === 0x3a && !quoted) return index;
+ }
+ return -1;
+}
+
+/**
+ * @param truncated Whether `bytes` is a prefix of the file, in which case a
+ * partial character at the end is a cut rather than a bad encoding.
+ * @param whole The complete file, when `bytes` is a prefix of it. A declaration
+ * can sit anywhere, so a preview that looked for one inside its own slice
+ * reported an error for a file the attachment itself decodes.
+ */
+export function decodeTextAttachmentBytes(
+ bytes: Uint8Array,
+ fileName = "",
+ truncated = false,
+ whole: Uint8Array = bytes,
+): string {
+ // A BOM is a declaration too, so it decodes as strictly as the rest: an odd
+ // trailing byte or an unpaired surrogate is corrupt, not readable.
+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
+ return decodeWithCharset(bytes.subarray(2), "utf-16le", fileName, truncated);
+ }
+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
+ return decodeWithCharset(bytes.subarray(2), "utf-16be", fileName, truncated);
+ }
+ // A document that states its own encoding decides before UTF-8 is tried:
+ // bytes that happen to be valid UTF-8 would otherwise decode into different
+ // characters than the file says it holds. An XML prolog is that statement by
+ // specification, a gettext header and a vCard property parameter by format,
+ // and all three are written by the exporter rather than typed by a person.
+ //
+ // A mail Content-Type is the exception and stays a fallback below: clients
+ // mislabel 8-bit mail constantly, so the bytes being valid UTF-8 is the
+ // better evidence there.
+ const xmlEncoding = declaredXmlEncoding(whole);
+ if (xmlEncoding) {
+ return decodeWithCharset(bytes, xmlEncoding, fileName, truncated);
+ }
+ const gettextCharset = declaredGettextCharset(whole, fileName);
+ if (gettextCharset) {
+ return decodeWithCharset(bytes, gettextCharset, fileName, truncated);
+ }
+ const vCardCharsets = declaredVCardCharsets(whole, fileName);
+ if (vCardCharsets.length > 0) {
+ // CHARSET is a property parameter whatever the count, so one declaration
+ // speaks for its own property and not for the file: an export holding a 2.1
+ // card beside a 3.0 one, which cannot declare anything, read the whole
+ // bundle in the 2.1 card's charset and turned the other into mojibake.
+ // A preview reads the same way, so what it shows is what gets sent.
+ const perProperty = decodeVCardPerProperty(bytes, truncated, fileName);
+ if ("text" in perProperty) {
+ return perProperty.text;
+ }
+ // A declaration the reading could not honour has to be reported, not read
+ // past. Falling through to UTF-8 accepted the card whenever its other
+ // properties happened to be ASCII, so one unsupported charset was refused
+ // on a card that declared it alone and ignored on a card that declared a
+ // second one beside it. The single-declaration case reports the same
+ // obstacle through the whole-unit reading below.
+ if (perProperty.failure && vCardCharsets.length > 1) {
+ throw perProperty.failure;
+ }
+ }
+ if (vCardCharsets.length === 1) {
+ // Only reached when a property broke its own declaration or named a charset
+ // with no decoder here, so the card is read as one unit as it used to be.
+ return decodeWithCharset(bytes, vCardCharsets[0]!, fileName, truncated);
+ }
+ try {
+ // A truncated read decodes with stream:true so the character the slice cut
+ // in half is dropped rather than raising. A whole file gets no such licence:
+ // there, a dangling lead byte is a bad encoding and has to be reported.
+ return new TextDecoder("utf-8", { fatal: true }).decode(bytes, {
+ stream: truncated,
+ });
+ } catch {
+ // An 8-bit mail says which charset it is, so honour it rather than refusing
+ // a standards-valid message. Tried only here, so a modern one is never
+ // remapped by a stale declaration. A vCard was tried above and only reaches
+ // this line when it named more than one charset.
+ const declared = vCardCharsets.length
+ ? vCardCharsets
+ : declaredEmailCharsets(whole, fileName);
+ if (declared.length === 1) {
+ return decodeWithCharset(bytes, declared[0]!, fileName, truncated);
+ }
+ if (declared.length > 1) {
+ // Parts in different encodings: decoding the container as one unit would
+ // corrupt all but one of them, and this is not a MIME parser.
+ throw new UndecodableTextError(
+ fileName,
+ `It declares more than one charset (${declared.join(", ")}), and is read as one unit.`,
+ );
+ }
+ // Otherwise a legacy code page, but which one is not knowable from the
+ // bytes: the same byte is a different letter in windows-1252, windows-1251
+ // and Shift-JIS. Guessing sends confident mojibake, so say so instead.
+ throw new UndecodableTextError(fileName);
+ }
+}
+
+/** Bytes that are not UTF-8 and carry no marker saying what they are. */
+export class UndecodableTextError extends Error {
+ constructor(fileName: string, reason?: string) {
+ super(
+ `${fileName || "This file"} is not UTF-8 text. ${
+ reason ?? "It looks like a legacy code page."
+ } Convert it to UTF-8 before attaching it.`,
+ );
+ this.name = "UndecodableTextError";
+ }
+}
+
+const OLE_COMPOUND_FILE_MAGIC = [
+ 0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1,
+];
+
+/** Legacy Word `.dot` and PowerPoint `.pot` templates are OLE compound files,
+ * unlike Graphviz `.dot` and gettext `.pot`. */
+export async function isBinaryOfficeTemplate(file: File): Promise<boolean> {
+ if (!/\.(?:dot|pot)$/i.test(file.name)) {
+ return false;
+ }
+ const header = new Uint8Array(await file.slice(0, 8).arrayBuffer());
+ return (
+ header.length === 8 &&
+ OLE_COMPOUND_FILE_MAGIC.every((byte, index) => header[index] === byte)
+ );
+}
+
+/** Decode editor text, including the BOM emitted by Windows Registry Editor. */
+export async function readTextAttachment(file: File): Promise<string> {
+ const bytes = new Uint8Array(await file.arrayBuffer());
+ return decodeTextAttachmentBytes(bytes, file.name);
+}
+
+// Dropped with the File itself, so a removed attachment retains nothing.
+const decodedOnce = new WeakMap<File, string>();
+
+/** Decode once per file. The composer decodes while attaching, to report a bad
+ * encoding there, and sending the same file must not read all of it again. */
+export async function readTextAttachmentOnce(file: File): Promise<string> {
+ const cached = decodedOnce.get(file);
+ if (cached !== undefined) {
+ return cached;
+ }
+ const text = await readTextAttachment(file);
+ decodedOnce.set(file, text);
+ return text;
+}
+
// MIME is unreliable for source files, so match by extension too.
export const TEXT_ATTACHMENT_ACCEPT = [
- "text/plain,text/markdown,text/csv,text/xml,text/json,text/css",
- "application/json,application/xml,image/svg+xml",
+ "text/plain,text/markdown,text/csv,text/tab-separated-values,text/xml,text/json,text/css",
+ "text/vtt,application/x-subrip,text/x-log,text/calendar,text/vcard,message/rfc822",
+ "application/json,application/xml,application/yaml,application/toml,image/svg+xml",
TEXT_ATTACHMENT_EXTENSIONS.join(","),
].join(",");
diff --git a/studio/frontend/src/features/chat/utils/clipboard-files.ts b/studio/frontend/src/features/chat/utils/clipboard-files.ts
index 211e628beb5..95433e0d25b 100644
--- a/studio/frontend/src/features/chat/utils/clipboard-files.ts
+++ b/studio/frontend/src/features/chat/utils/clipboard-files.ts
@@ -11,8 +11,14 @@ import {
const MAX_NATIVE_IMAGE_DIMENSION = 8192;
const MAX_NATIVE_IMAGE_RGBA_BYTES = 64 * 1024 * 1024;
-const MAX_CLIPBOARD_BYTES = MAX_AUDIO_SIZE;
const MAX_CLIPBOARD_NON_AUDIO_BYTES = 20 * 1024 * 1024;
+// Mirrors MAX_CLIPBOARD_VIDEO_BYTES in native_clipboard.rs. The native reader
+// takes a pasted clip up to this, so refusing it here threw away a file it had
+// already read and encoded, and dropped the rest of the paste with it.
+const MAX_CLIPBOARD_VIDEO_BYTES = 75_497_280;
+// The largest a single file may be, so the total never refuses one that the
+// per-file limits allow on its own. Matches MAX_CLIPBOARD_TOTAL_BYTES.
+const MAX_CLIPBOARD_BYTES = MAX_CLIPBOARD_VIDEO_BYTES;
const MAX_CLIPBOARD_FILES = 8;
type ClipboardPasteEvent = {
@@ -63,9 +69,13 @@ async function readNativeClipboardFiles(): Promise<File[]> {
) {
return [];
}
+ // Per kind, as the native reader does: what it hands over, this accepts,
+ // and the composer's own gates give the user the reason for anything else.
const maxFileBytes = file.mimeType.startsWith("audio/")
? MAX_AUDIO_SIZE
- : MAX_CLIPBOARD_NON_AUDIO_BYTES;
+ : file.mimeType.startsWith("video/")
+ ? MAX_CLIPBOARD_VIDEO_BYTES
+ : MAX_CLIPBOARD_NON_AUDIO_BYTES;
if (file.base64.length > Math.ceil((maxFileBytes * 4) / 3) + 4) {
return [];
}
diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.ts
index 04f719f769c..5fb7c1a96b6 100644
--- a/studio/frontend/src/features/native-intents/drop-paths.ts
+++ b/studio/frontend/src/features/native-intents/drop-paths.ts
@@ -5,7 +5,10 @@ import {
OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS,
isOpenDocumentAttachmentName,
} from "../chat/open-document-accept.ts";
-import { TEXT_ATTACHMENT_EXTENSIONS } from "../chat/text-attachment-accept.ts";
+import {
+ TEXT_ATTACHMENT_EXTENSIONS,
+ isTextAttachmentName,
+} from "../chat/text-attachment-accept.ts";
import { RAG_UPLOAD_ACCEPT } from "../rag/types/rag.ts";
const DOC_EXTS = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>
@@ -23,11 +26,13 @@ export function isComposerAttachmentName(path: string): boolean {
}
function isTextDropName(path: string): boolean {
- // Rust classifies on Path::extension, which a dotfile like ".env" has none of.
const name = nativeFileName(path).toLowerCase();
+ if (!name.includes(".") && isTextAttachmentName(name)) {
+ return true;
+ }
+ // A dotfile like ".env" has no extension and remains unsupported.
const dot = name.lastIndexOf(".");
- if (dot <= 0) return false;
- return TEXT_EXTS.includes(name.slice(dot));
+ return dot > 0 && TEXT_EXTS.includes(name.slice(dot));
}
/** Vision chat attachments; keep in sync with `shared-composer` `IMAGE_ACCEPT`. */
@@ -38,7 +43,8 @@ const IMAGE_EXTS = CHAT_IMAGE_DROP_ACCEPT.split(",").map((ext) =>
);
/** Chat audio attachments; keep in sync with `audio-attachment-adapter.ts` `accept`. */
-export const CHAT_AUDIO_DROP_ACCEPT = ".wav,.mp3,.m4a,.ogg,.oga,.flac";
+export const CHAT_AUDIO_DROP_ACCEPT =
+ ".wav,.mp3,.m4a,.ogg,.oga,.opus,.flac,.aac,.aiff,.aif,.aifc,.caf,.wma,.amr,.mp2";
const AUDIO_EXTS = CHAT_AUDIO_DROP_ACCEPT.split(",").map((ext) =>
ext.trim().toLowerCase(),
@@ -47,7 +53,8 @@ const AUDIO_EXTS = CHAT_AUDIO_DROP_ACCEPT.split(",").map((ext) =>
/** Chat video attachments; keep in sync with `native_path_policy.rs`
* `VIDEO_ATTACHMENT_EXTS`. llama-server decodes with ffmpeg, so this is what
* ffmpeg reads, not what the webview can play. */
-export const CHAT_VIDEO_DROP_ACCEPT = ".mp4,.mov,.webm,.mkv,.avi";
+export const CHAT_VIDEO_DROP_ACCEPT =
+ ".mp4,.m4v,.mov,.webm,.mkv,.avi,.mpg,.mpeg,.wmv,.flv,.3gp,.ogv";
const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) =>
ext.trim().toLowerCase(),
diff --git a/studio/frontend/src/features/video/reference-budget.ts b/studio/frontend/src/features/video/reference-budget.ts
index 6b69328c07f..97b13b53382 100644
--- a/studio/frontend/src/features/video/reference-budget.ts
+++ b/studio/frontend/src/features/video/reference-budget.ts
@@ -1,6 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import { AUDIO_PICKER_ACCEPT, isAudioAttachmentFile } from "../../lib/audio-utils.ts";
+import { VIDEO_ACCEPT, isVideoFile } from "../../lib/video-utils.ts";
+
/** MiniMax-H3's combined budget across picture, video and standalone-audio references. */
export const MAX_H3_REFERENCES = 12;
@@ -42,12 +45,48 @@ export const MAX_REFERENCE_BYTES: Record<ReferenceKind, number> = {
audio: rawLimitFor(32 * 1024 * 1024),
};
+/** What a reference file dialog should offer, per kind. Extensions ride along
+ * with the MIME types: a browser answers "" for wma, amr, caf and several
+ * others, and `${kind}/*` alone greys those out of the dialog. The audio list
+ * is the picker one, .3gp included, because the picker reads a recording's
+ * tracks once it has the file and a clip is refused then. */
+export const REFERENCE_PICKER_ACCEPT: Record<ReferenceKind, string> = {
+ video: VIDEO_ACCEPT,
+ audio: AUDIO_PICKER_ACCEPT,
+};
+
+/**
+ * The same policy for the drop zone, extensions only.
+ *
+ * A drop zone filters on the name before the classifier can look at the file,
+ * so a list narrower than the dialog's refuses what the button accepts. It
+ * carries no MIME types because the zone matches names and puts this list
+ * verbatim into the message it shows when it turns a file away.
+ */
+export const REFERENCE_DROP_ACCEPT: Record<ReferenceKind, string> = {
+ video: extensionsOf(REFERENCE_PICKER_ACCEPT.video),
+ audio: extensionsOf(REFERENCE_PICKER_ACCEPT.audio),
+};
+
+function extensionsOf(accept: string): string {
+ return accept
+ .split(",")
+ .map((entry) => entry.trim())
+ .filter((entry) => entry.startsWith("."))
+ .join(",");
+}
+
/** Why this file cannot be staged as a reference, or null when it can. */
export function referenceFileRejection(
kind: ReferenceKind,
- file: { type: string; size: number },
+ file: { type: string; size: number; name?: string },
): string | null {
- if (!file.type.startsWith(`${kind}/`)) {
+ // Name as well as MIME, matching the accept list above and the native drop:
+ // the same recording chosen through the button used to be refused for the
+ // empty type the browser gave it.
+ const named = { type: file.type, name: file.name ?? "" };
+ const matches = kind === "video" ? isVideoFile(named) : isAudioAttachmentFile(named);
+ if (!matches) {
return `Please choose ${kind === "video" ? "a video" : "an audio"} file`;
}
if (file.size > MAX_REFERENCE_BYTES[kind]) {
diff --git a/studio/frontend/src/features/video/reference-picker.tsx b/studio/frontend/src/features/video/reference-picker.tsx
index ae13cf512c3..155ee09df23 100644
--- a/studio/frontend/src/features/video/reference-picker.tsx
+++ b/studio/frontend/src/features/video/reference-picker.tsx
@@ -11,15 +11,17 @@ import {
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
-import {
- CHAT_AUDIO_DROP_ACCEPT,
- CHAT_VIDEO_DROP_ACCEPT,
-} from "@/features/native-intents/drop-paths";
import { useNativeFileDrop } from "@/features/native-intents";
import { cn } from "@/lib/utils";
import { toast } from "@/lib/toast";
-import { createReferenceSelectionGate, readReferenceFile } from "./reference-budget";
+import {
+ REFERENCE_DROP_ACCEPT,
+ REFERENCE_PICKER_ACCEPT,
+ createReferenceSelectionGate,
+ readReferenceFile,
+} from "./reference-budget";
+import { classifiedAttachmentFile } from "@/lib/video-utils";
/** One staged reference file: the data URL the request carries, plus its name for the chip. */
export interface ReferenceMedia {
@@ -91,9 +93,13 @@ export function ReferenceMediaPicker({
}, [value, gate]);
const readFile = useCallback(
- (file: File | undefined | null) => {
- if (!file) return;
+ async (picked: File | undefined | null) => {
+ if (!picked) return;
const claim = gate.begin();
+ // A .3gp is a recording or a clip and its name says neither, so read the
+ // container's tracks before the kind check, as chat and compare do.
+ const file = await classifiedAttachmentFile(picked);
+ if (!claim.isCurrent()) return;
readReferenceFile(kind, file, {
onLoaded: (dataUrl) => {
if (!claim.isCurrent()) return;
@@ -119,8 +125,8 @@ export function ReferenceMediaPicker({
// Tauri suppresses the webview's own drop events, so the handlers below never
// fire on the desktop app; this claims the OS drop for the button (#9036).
const { ref: dropRef, dragging, dragHandlers } = useNativeFileDrop({
- onFiles: (files) => readFile(files[0]),
- accept: kind === "video" ? CHAT_VIDEO_DROP_ACCEPT : CHAT_AUDIO_DROP_ACCEPT,
+ onFiles: (files) => void readFile(files[0]),
+ accept: REFERENCE_DROP_ACCEPT[kind],
multiple: false,
});
@@ -184,9 +190,9 @@ export function ReferenceMediaPicker({
<input
ref={inputRef}
type="file"
- accept={`${kind}/*`}
+ accept={REFERENCE_PICKER_ACCEPT[kind]}
className="hidden"
- onChange={(e) => readFile(e.target.files?.[0])}
+ onChange={(e) => void readFile(e.target.files?.[0])}
/>
</button>
);
diff --git a/studio/frontend/src/lib/audio-utils.ts b/studio/frontend/src/lib/audio-utils.ts
index 0a443b8d21b..bdf95412d93 100644
--- a/studio/frontend/src/lib/audio-utils.ts
+++ b/studio/frontend/src/lib/audio-utils.ts
@@ -1,7 +1,41 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-export const AUDIO_ACCEPT = "audio/wav,audio/mpeg,audio/webm,audio/ogg,audio/flac,audio/mp4";
+// Containers the backend can reach wav/mp3 from: wav and mp3 pass through, the
+// rest decode via libsndfile, PyAV's bundled FFmpeg, or finally librosa.
+export const AUDIO_ACCEPT =
+ "audio/wav,audio/mpeg,audio/webm,audio/ogg,audio/opus,audio/flac,audio/mp4,audio/aac,audio/aiff,audio/x-aiff,audio/x-caf,audio/x-ms-wma,audio/amr,audio/3gpp";
+// Browsers report an empty or wrong type for several of these containers, so
+// every surface that takes audio matches the name as well as the MIME.
+export const AUDIO_ACCEPT_EXTENSIONS =
+ ".wav,.mp3,.m4a,.ogg,.oga,.opus,.flac,.aac,.aiff,.aif,.aifc,.caf,.wma,.amr,.mp2";
+/** What claims a file as audio: the MIME list plus those extensions. Keep .3gp
+ * out of it. A recording and a clip share that extension, and the composer's
+ * audio adapter is matched before its video one, so claiming the name here
+ * would take every 3GP video as audio. */
+export const AUDIO_ATTACHMENT_ACCEPT = `${AUDIO_ACCEPT},audio/x-m4a,${AUDIO_ACCEPT_EXTENSIONS}`;
+/**
+ * What a file dialog should offer.
+ *
+ * Wider than the accept above, because a dialog only decides what is selectable
+ * and a platform that maps .3gp to video/3gpp, or to nothing, greys out a voice
+ * recording the surface would otherwise take. The tracks are read once the file
+ * is in hand, and a real clip is refused then.
+ */
+export const AUDIO_PICKER_ACCEPT = `${AUDIO_ATTACHMENT_ACCEPT},.3gp`;
+
+/** Whether a dropped or pasted file is audio, by MIME or, failing that, name. */
+export function isAudioAttachmentFile(file: {
+ name: string;
+ type: string;
+}): boolean {
+ if (/^audio\//i.test(file.type)) {
+ return true;
+ }
+ const name = file.name.toLowerCase();
+ return AUDIO_ACCEPT_EXTENSIONS.split(",").some((ext) => name.endsWith(ext));
+}
+
// Keep in sync with STT_AUDIO_RAW_MAX_BYTES in the backend upload limits.
const MAX_AUDIO_SIZE_MB = 25;
export const MAX_AUDIO_SIZE = MAX_AUDIO_SIZE_MB * 1024 * 1024;
diff --git a/studio/frontend/src/lib/video-utils.ts b/studio/frontend/src/lib/video-utils.ts
index 0c3f1c68817..671e29c3c76 100644
--- a/studio/frontend/src/lib/video-utils.ts
+++ b/studio/frontend/src/lib/video-utils.ts
@@ -5,7 +5,7 @@
* what ffmpeg reads, not what the webview can play. Extensions ride along
* because MIME is unreliable for mkv and some mov files. */
export const VIDEO_ACCEPT =
- "video/mp4,video/quicktime,video/webm,video/x-matroska,video/x-msvideo,.mp4,.mov,.webm,.mkv,.avi";
+ "video/mp4,video/x-m4v,video/quicktime,video/webm,video/x-matroska,video/x-msvideo,video/mpeg,video/x-ms-wmv,video/x-flv,video/3gpp,video/ogg,.mp4,.m4v,.mov,.webm,.mkv,.avi,.mpg,.mpeg,.wmv,.flv,.3gp,.ogv";
// Matches _MAX_VIDEO_B64_CHARS in the backend, so the composer does not accept
// a clip the route refuses. The native reader's cap is a higher backstop.
@@ -24,10 +24,17 @@ export function getVideoSizeError(size: number): string | null {
// browser must reach the route as the same mime type.
const VIDEO_MIME_BY_EXTENSION: Record<string, string> = {
".mp4": "video/mp4",
+ ".m4v": "video/x-m4v",
".mov": "video/quicktime",
".webm": "video/webm",
".mkv": "video/x-matroska",
".avi": "video/x-msvideo",
+ ".mpg": "video/mpeg",
+ ".mpeg": "video/mpeg",
+ ".wmv": "video/x-ms-wmv",
+ ".flv": "video/x-flv",
+ ".3gp": "video/3gpp",
+ ".ogv": "video/ogg",
};
const VIDEO_EXTENSIONS = Object.keys(VIDEO_MIME_BY_EXTENSION);
@@ -54,10 +61,216 @@ export function videoMimeForFile(file: File): string {
/** Whether a picked file is a video. mkv and some mov files arrive with an
* empty MIME type, hence the extension fallback. */
-export function isVideoFile(file: File): boolean {
+export function isVideoFile(file: { name: string; type: string }): boolean {
if (VIDEO_MIME_RE.test(file.type)) {
return true;
}
+ // The extension fallback below claims .3gp, which a recording shares with a
+ // clip. Something that read the tracks has already said which this is.
+ if (/^audio\//i.test(file.type)) {
+ return false;
+ }
const name = file.name.toLowerCase();
return VIDEO_EXTENSIONS.some((ext) => name.endsWith(ext));
}
+
+/** Payloads of every box of the given type at this level. Mirrors
+ * `bmff_box_payloads` in native_path_policy.rs. */
+function bmffBoxPayloads(data: Uint8Array, wanted: string): Uint8Array[] {
+ const payloads: Uint8Array[] = [];
+ const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+ let offset = 0;
+ while (data.length - offset >= 8) {
+ const size32 = view.getUint32(offset);
+ const type = String.fromCharCode(
+ data[offset + 4]!,
+ data[offset + 5]!,
+ data[offset + 6]!,
+ data[offset + 7]!,
+ );
+ let headerSize = 8;
+ let boxSize = size32;
+ if (size32 === 0) {
+ // Runs to the end of this level.
+ boxSize = data.length - offset;
+ } else if (size32 === 1) {
+ // A 64-bit size follows the type.
+ if (data.length - offset < 16) break;
+ const size64 = view.getBigUint64(offset + 8);
+ if (size64 > BigInt(Number.MAX_SAFE_INTEGER)) break;
+ headerSize = 16;
+ boxSize = Number(size64);
+ }
+ if (boxSize < headerSize || boxSize > data.length - offset) break;
+ if (type === wanted) {
+ payloads.push(data.subarray(offset + headerSize, offset + boxSize));
+ }
+ offset += boxSize;
+ }
+ return payloads;
+}
+
+type BmffTracks = { audio: boolean; video: boolean };
+
+/** The handler types a moov box's tracks declare. */
+function tracksInMoov(moov: Uint8Array, found: BmffTracks): void {
+ for (const trak of bmffBoxPayloads(moov, "trak")) {
+ for (const mdia of bmffBoxPayloads(trak, "mdia")) {
+ for (const hdlr of bmffBoxPayloads(mdia, "hdlr")) {
+ if (hdlr.length < 12) continue;
+ const handler = String.fromCharCode(
+ hdlr[8]!,
+ hdlr[9]!,
+ hdlr[10]!,
+ hdlr[11]!,
+ );
+ if (handler === "soun") found.audio = true;
+ else if (handler === "vide") found.video = true;
+ }
+ }
+ }
+}
+
+/** Whether a 3GP container has audio tracks and no video tracks. Mirrors
+ * `is_audio_only_3gp` in native_path_policy.rs. */
+export function isAudioOnly3gpBytes(raw: Uint8Array): boolean {
+ const found: BmffTracks = { audio: false, video: false };
+ for (const moov of bmffBoxPayloads(raw, "moov")) {
+ tracksInMoov(moov, found);
+ }
+ return found.audio && !found.video;
+}
+
+// A track table runs to kilobytes; anything this large is not one, and reading
+// it would be the memory problem the box walk exists to avoid.
+const MAX_MOOV_BYTES = 8 * 1024 * 1024;
+// A container holds a handful of these: ftyp, moov, mdat, and maybe free or
+// mfra. A file that reports thousands is malformed, and walking it would be
+// one slice per box.
+const MAX_TOP_LEVEL_BOXES = 64;
+
+/**
+ * Which track kinds a container declares, without holding it.
+ *
+ * The tracks live in `moov`, and the samples in `mdat` beside it, so reading the
+ * file to reach a handler retains the whole clip: a 64 MB one costs 64 MB, and a
+ * dropped batch costs that per file at once. This walks the top-level boxes
+ * through slices and reads only `moov`, which is the same walk `bmffBoxPayloads`
+ * does, one level up.
+ */
+async function read3gpTracks(file: File): Promise<BmffTracks> {
+ const found: BmffTracks = { audio: false, video: false };
+ let offset = 0;
+ for (let box = 0; box < MAX_TOP_LEVEL_BOXES && offset + 8 <= file.size; box++) {
+ const header = new Uint8Array(
+ await file.slice(offset, offset + 16).arrayBuffer(),
+ );
+ if (header.length < 8) break;
+ const view = new DataView(
+ header.buffer,
+ header.byteOffset,
+ header.byteLength,
+ );
+ const size32 = view.getUint32(0);
+ const type = String.fromCharCode(
+ header[4]!,
+ header[5]!,
+ header[6]!,
+ header[7]!,
+ );
+ let headerSize = 8;
+ let boxSize = size32;
+ if (size32 === 0) {
+ boxSize = file.size - offset;
+ } else if (size32 === 1) {
+ if (header.length < 16) break;
+ const size64 = view.getBigUint64(8);
+ if (size64 > BigInt(Number.MAX_SAFE_INTEGER)) break;
+ headerSize = 16;
+ boxSize = Number(size64);
+ }
+ if (boxSize < headerSize || boxSize > file.size - offset) break;
+ if (type === "moov") {
+ if (boxSize - headerSize > MAX_MOOV_BYTES) break;
+ const moov = new Uint8Array(
+ await file.slice(offset + headerSize, offset + boxSize).arrayBuffer(),
+ );
+ tracksInMoov(moov, found);
+ }
+ offset += boxSize;
+ }
+ return found;
+}
+
+/**
+ * Whether a file's own tracks have to be read before it can be classified.
+ * Cheap and synchronous, so a surface can keep its existing path for everything
+ * else.
+ *
+ * The extension alone, and nothing about the MIME type or the size.
+ *
+ * Not the type, because it comes from the same ambiguous extension: a platform
+ * that reports audio/3gpp for a recording reports it for a clip as well, and
+ * trusting that sent the clip down the audio path.
+ *
+ * Not the size, because the one condition here sat at the composer's video cap
+ * while the video reference surface accepts a larger file, so a recording in
+ * between skipped inspection entirely. A ceiling that has to track every
+ * surface's limit will fall behind one of them, and the walk below is bounded
+ * by box count and by the size of the track table rather than by the file.
+ */
+export function needsAttachmentTrackInspection(file: File): boolean {
+ return /\.3gp$/i.test(file.name);
+}
+
+/**
+ * The file an attachment surface should classify, with an audio-only 3GP
+ * restamped as audio.
+ *
+ * A voice recording and a clip share the .3gp extension, and the browser
+ * answers "" or video/3gpp for both, so the name alone sends the recording to
+ * whichever surface claims video: rejected outright on an audio model, and fed
+ * to ffmpeg as frames on a video one. The native readers already read the BMFF
+ * handlers and stamp audio/3gpp, so do the same before an adapter is picked.
+ * Everything else is returned untouched, so this costs one predicate per file.
+ */
+export async function classifiedAttachmentFile(file: File): Promise<File> {
+ if (!needsAttachmentTrackInspection(file)) {
+ return file;
+ }
+ let tracks: BmffTracks;
+ try {
+ tracks = await read3gpTracks(file);
+ } catch {
+ // An unreadable file is left as it came; the surface reports the read.
+ return file;
+ }
+ // Both directions, because the browser's answer comes from the same ambiguous
+ // extension: a platform that maps .3gp to audio/3gpp says so for a clip too,
+ // and the audio adapter is matched before the video one. Tracks it cannot
+ // read decide nothing, so the file is left as it came.
+ const corrected = tracks.video
+ ? "video/3gpp"
+ : tracks.audio
+ ? "audio/3gpp"
+ : null;
+ if (corrected === null || corrected === file.type) {
+ return file;
+ }
+ return new File([file], file.name, {
+ type: corrected,
+ lastModified: file.lastModified,
+ });
+}
+
+/** The same restamping across a picked or dropped batch, one file at a time so
+ * a drop of several never has more than one container's boxes in hand. */
+export async function classifiedAttachmentFiles(
+ files: FileList | readonly File[],
+): Promise<File[]> {
+ const classified: File[] = [];
+ for (const file of Array.from(files)) {
+ classified.push(await classifiedAttachmentFile(file));
+ }
+ return classified;
+}
diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.ts
index c579fbf9ae5..ccb6adb9ebd 100644
--- a/studio/frontend/tests/attachment-preview-text.test.ts
+++ b/studio/frontend/tests/attachment-preview-text.test.ts
@@ -196,6 +196,43 @@ test("readAttachmentText reads a bounded slice of a large text file", async () =
assert.equal(truncateAttachmentPreviewText(text).truncated, true);
});
+test("readAttachmentText previews UTF-16 registry exports as decoded text", async () => {
+ const text =
+ "Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Test]";
+ const utf16le = new Uint8Array(2 + text.length * 2);
+ utf16le.set([0xff, 0xfe]);
+ for (let index = 0; index < text.length; index += 1) {
+ const codeUnit = text.charCodeAt(index);
+ utf16le[2 + index * 2] = codeUnit & 0xff;
+ utf16le[3 + index * 2] = codeUnit >>> 8;
+ }
+ const file = new File([utf16le], "export.reg");
+
+ assert.deepEqual(await readAttachmentText(file, file.name, file.type), {
+ label: null,
+ text,
+ truncated: false,
+ });
+});
+
+test("readAttachmentText previews gettext catalogs in their declared charset", async () => {
+ const before =
+ 'msgid ""\nmsgstr ""\n"Content-Type: text/plain; charset=ISO-8859-1\\n"\n\nmsgid "coffee"\nmsgstr "caf';
+ const after = '"\n';
+ const encoded = Uint8Array.from([
+ ...new TextEncoder().encode(before),
+ 0xe9,
+ ...new TextEncoder().encode(after),
+ ]);
+ const file = new File([encoded], "messages.po");
+
+ assert.deepEqual(await readAttachmentText(file, file.name, file.type), {
+ label: null,
+ text: `${before}é${after}`,
+ truncated: false,
+ });
+});
+
test("readAttachmentText reads a bounded slice of a large html file", async () => {
const oversized = new File(
[`<p>${"b".repeat(2_000_000)}</p>`],
@@ -919,3 +956,35 @@ test("parseAttachmentText keeps an unterminated tag as plain text", () => {
truncated: false,
});
});
+
+test("a preview is never stricter than the adapter that took the file", async () => {
+ // .html belongs to the HTML adapter, which sends a legacy page happily. The
+ // preview went through the strict text decoder and threw on the same file,
+ // so opening an attachment that had already been accepted failed.
+ const head = new TextEncoder().encode(
+ '<!doctype html><meta charset="windows-1252"><body>Caf',
+ );
+ const bytes = new Uint8Array([
+ ...head,
+ 0xe9,
+ ...new TextEncoder().encode("</body>"),
+ ]);
+ const page = new File([bytes], "page.html", { type: "text/html" });
+ const preview = await readAttachmentText(page, page.name, page.type);
+ assert.equal(typeof preview.text, "string");
+ assert.ok(preview.text.includes("Caf"));
+
+ // A file the text adapter does own stays strict: mojibake reaching the model
+ // is worse than a message saying the encoding could not be read.
+ const { UndecodableTextError } = await import(
+ "../src/features/chat/text-attachment-accept.ts"
+ );
+ await assert.rejects(
+ readAttachmentText(
+ new File([bytes], "notes.srt"),
+ "notes.srt",
+ "text/plain",
+ ),
+ (error: Error) => error instanceof UndecodableTextError,
+ );
+});
diff --git a/studio/frontend/tests/native-drop-paths.test.ts b/studio/frontend/tests/native-drop-paths.test.ts
index 855951fcc77..51383087d4e 100644
--- a/studio/frontend/tests/native-drop-paths.test.ts
+++ b/studio/frontend/tests/native-drop-paths.test.ts
@@ -11,7 +11,27 @@ import {
} from "../src/features/chat/open-document-accept.ts";
import {
TEXT_ATTACHMENT_ACCEPT,
+ TEXT_ATTACHMENT_BASENAMES,
TEXT_ATTACHMENT_EXTENSIONS,
+ isBinaryPropertyList,
+ isBinaryTrackerModule,
+ MAX_TEXT_ATTACHMENT_BYTES,
+ decodeTextAttachmentBytes,
+ isTextAttachmentName,
+} from "../src/features/chat/text-attachment-accept.ts";
+import {
+ AUDIO_ATTACHMENT_ACCEPT,
+ AUDIO_PICKER_ACCEPT,
+ isAudioAttachmentFile,
+} from "../src/lib/audio-utils.ts";
+import {
+ isBinaryOfficeTemplate,
+ isBinaryVobSubSubtitle,
+ isCompiledFortranModule,
+ pickerAcceptForTextBasenames,
+ readTextAttachment,
+ readTextAttachmentOnce,
+ UndecodableTextError,
} from "../src/features/chat/text-attachment-accept.ts";
import {
dequeueNativeAttachments,
@@ -29,7 +49,13 @@ import type { NativeIntent } from "../src/features/native-intents/types.ts";
import { RAG_UPLOAD_ACCEPT } from "../src/features/rag/types/rag.ts";
import { MAX_REFERENCE_BYTES } from "../src/features/video/reference-budget.ts";
import { AUDIO_ACCEPT } from "../src/lib/audio-utils.ts";
-import { VIDEO_ACCEPT } from "../src/lib/video-utils.ts";
+import {
+ VIDEO_ACCEPT,
+ classifiedAttachmentFile,
+ classifiedAttachmentFiles,
+ isAudioOnly3gpBytes,
+ isVideoFile,
+} from "../src/lib/video-utils.ts";
import { registerBundlerResolver } from "./helpers/kit.ts";
registerBundlerResolver();
@@ -51,6 +77,9 @@ const RUST_VIDEO_ATTACHMENT_EXTS_RE =
/VIDEO_ATTACHMENT_EXTS[^=]*=\s*&\[([^\]]+)\]/s;
const RUST_TEXT_ATTACHMENT_EXTS_RE =
/TEXT_ATTACHMENT_EXTS[^=]*=\s*&\[([^\]]+)\]/s;
+const RUST_TEXT_ATTACHMENT_NAMES_RE =
+ /TEXT_ATTACHMENT_NAMES[^=]*=\s*&\[([^\]]+)\]/s;
+const PICKER_BASENAME_CALL_RE = /pickerAcceptForTextBasenames\(enabledAccept\)/;
const RUST_VIDEO_MIME_RE = /Some\("(video\/[^"]+)"\)/g;
const DOTTED_EXTENSION_RE = /"(\.[^"]+)"/g;
const RUST_EXTENSION_RE = /"([^"]+)"/g;
@@ -678,68 +707,1906 @@ test("a dropped source file attaches instead of being refused", () => {
assert.equal(mixed.kind, "docs");
});
-test("RAG documents keep their existing route", () => {
- for (const path of ["/docs/notes.txt", "/docs/notes.md", "/docs/paper.pdf"]) {
+test("the conventional extensionless Containerfile attaches as text", () => {
+ for (const path of ["/project/Containerfile", "C:\\project\\CONTAINERFILE"]) {
assert.equal(classifyDropPaths([path]).kind, "docs", path);
- assert.equal(isComposerAttachmentName(path), false, path);
+ assert.equal(isComposerAttachmentName(path), true, path);
}
});
-test("a dotfile is not mistaken for a source file", () => {
- // Rust classifies on Path::extension, which ".env" does not have.
- for (const path of ["/p/.env", "/p/.properties", "/p/.log"]) {
- assert.equal(isComposerAttachmentName(path), false, path);
- assert.equal(classifyDropPaths([path]).kind, "unsupported", path);
- }
- assert.equal(isComposerAttachmentName("/p/app.env"), true);
-});
+test("the picker remains able to show extensionless text basenames", () => {
+ assert.equal(pickerAcceptForTextBasenames(TEXT_ATTACHMENT_ACCEPT), "*");
+ assert.equal(
+ pickerAcceptForTextBasenames(".txt,text/plain"),
+ ".txt,text/plain",
+ );
-test("an unreadable type is still refused", () => {
- assert.equal(classifyDropPaths(["/docs/archive.zip"]).kind, "unsupported");
- assert.equal(classifyDropPaths(["/bin/tool.exe"]).kind, "unsupported");
+ const threadSource = readFileSync(
+ new URL("../src/components/assistant-ui/thread.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(threadSource, PICKER_BASENAME_CALL_RE);
});
-test("frontend and Rust accept the same dropped text extensions", () => {
- const docs = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>
- ext.trim().toLowerCase(),
- );
- const frontend = TEXT_ATTACHMENT_EXTENSIONS.map((ext) => ext.toLowerCase())
- .filter((ext) => !docs.includes(ext))
- .sort();
+test("frontend and Rust accept the same extensionless text names", () => {
const rustSource = readFileSync(
new URL("../../src-tauri/src/native_path_policy.rs", import.meta.url),
"utf8",
);
const rust = [
...(rustSource
- .match(RUST_TEXT_ATTACHMENT_EXTS_RE)?.[1]
+ .match(RUST_TEXT_ATTACHMENT_NAMES_RE)?.[1]
.matchAll(RUST_EXTENSION_RE) ?? []),
]
- .map((match) => `.${match[1]}`)
+ .map((match) => match[1])
.sort();
+ assert.deepEqual(rust, [...TEXT_ATTACHMENT_BASENAMES].sort());
+});
- assert.deepEqual(rust, frontend);
+test("binary property-list files are distinguished from text forms", async () => {
+ assert.equal(
+ await isBinaryPropertyList(new File(["bplist00payload"], "settings.plist")),
+ true,
+ );
+ assert.equal(
+ await isBinaryPropertyList(
+ new File(['<?xml version="1.0"?><plist/>'], "settings.plist"),
+ ),
+ false,
+ );
+ assert.equal(
+ await isBinaryPropertyList(new File(["bplist00payload"], "settings.txt")),
+ false,
+ );
+ assert.equal(
+ await isBinaryPropertyList(
+ new File(["bplist00payload"], "Localizable.strings"),
+ ),
+ true,
+ );
+ assert.equal(
+ await isBinaryPropertyList(
+ new File(['"hello" = "world";'], "Localizable.strings"),
+ ),
+ false,
+ );
});
-test("the composer adapter reads the shared text accept list", () => {
- const src = readFileSync(
+test("binary VobSub files are distinguished from text subtitles", async () => {
+ assert.equal(
+ await isBinaryVobSubSubtitle(
+ new File([new Uint8Array([0x00, 0x00, 0x01, 0xba, 0x44])], "movie.sub"),
+ ),
+ true,
+ );
+ assert.equal(
+ await isBinaryVobSubSubtitle(new File(["{1}{25}Hello|world"], "movie.sub")),
+ false,
+ );
+ assert.equal(
+ await isBinaryVobSubSubtitle(
+ new File([new Uint8Array([0x00, 0x00, 0x01, 0xba])], "movie.bin"),
+ ),
+ false,
+ );
+});
+
+test("tracker MOD binaries are distinguished from text module files", async () => {
+ const tracker = new Uint8Array(1084);
+ tracker.set(new TextEncoder().encode("M.K."), 1080);
+ assert.equal(
+ await isBinaryTrackerModule(new File([tracker], "track.mod")),
+ true,
+ );
+ const soundtracker = new Uint8Array(600 + 1024 + 8);
+ soundtracker[43] = 4; // Four big-endian words of sample data.
+ soundtracker[45] = 64;
+ soundtracker[470] = 1;
+ soundtracker[471] = 120;
+ assert.equal(
+ await isBinaryTrackerModule(new File([soundtracker], "classic.mod")),
+ true,
+ );
+ assert.equal(
+ await isBinaryTrackerModule(
+ new File(["module example.com/project\n".padEnd(1200, " ")], "go.mod"),
+ ),
+ false,
+ );
+ assert.equal(
+ await isBinaryTrackerModule(new File([tracker], "track.bin")),
+ false,
+ );
+});
+
+test("compiled Fortran modules are rejected, text .mod files are not", async () => {
+ // gfortran gzips its modules, so the tracker-header check never sees them.
+ const compiled = new Uint8Array([0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00]);
+ assert.equal(
+ await isCompiledFortranModule(new File([compiled], "kinds.mod")),
+ true,
+ );
+ assert.equal(
+ await isCompiledFortranModule(
+ new File(["module example.com/project\n"], "go.mod"),
+ ),
+ false,
+ );
+ // An uncompressed gfortran module is readable text and stays accepted.
+ assert.equal(
+ await isCompiledFortranModule(
+ new File(["GFORTRAN module version '15'\n"], "kinds.mod"),
+ ),
+ false,
+ );
+ assert.equal(
+ await isCompiledFortranModule(new File([compiled], "archive.gz")),
+ false,
+ );
+});
+
+test("a legacy code page subtitle is named, not guessed at", async () => {
+ // The same byte is a different letter in windows-1252, windows-1251 and
+ // Shift-JIS, so decoding it as any one of them sends confident mojibake.
+ const srt = new Uint8Array([
+ ...new TextEncoder().encode("1\n00:00:01,000 --> 00:00:02,000\nCaf"),
+ 0xe9, // valid in several code pages, invalid on its own in UTF-8
+ 0x0a,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([srt], "movie.srt")),
+ (error: Error) => {
+ assert.ok(error instanceof UndecodableTextError);
+ assert.match(error.message, /movie\.srt is not UTF-8 text/);
+ return true;
+ },
+ );
+});
+
+test("valid UTF-8 keeps its own decoding", async () => {
+ const text = await readTextAttachment(
+ new File([new TextEncoder().encode("Caf\u00e9 \u2014 na\u00efve")], "notes.srt"),
+ );
+ assert.equal(text, "Caf\u00e9 \u2014 na\u00efve");
+});
+
+test("a preview cut mid-character stays UTF-8", () => {
+ // The bounded preview slices at a byte offset, so the last character can be
+ // half-read. That is a truncation, not a legacy encoding.
+ const full = new TextEncoder().encode("caf\u00e9");
+ const cut = full.subarray(0, full.length - 1);
+ assert.equal(decodeTextAttachmentBytes(cut, "notes.txt", true), "caf");
+ // A whole file gets no such licence: a dangling lead byte is a bad encoding,
+ // not a cut, and dropping it silently would lose the character.
+ assert.throws(
+ () => decodeTextAttachmentBytes(cut, "notes.txt"),
+ (error: Error) => error instanceof UndecodableTextError,
+ );
+});
+
+test("legacy Office templates are rejected, text templates are not", async () => {
+ const ole = new Uint8Array([
+ 0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0x00, 0x00,
+ ]);
+ assert.equal(
+ await isBinaryOfficeTemplate(new File([ole], "report.dot")),
+ true,
+ );
+ assert.equal(await isBinaryOfficeTemplate(new File([ole], "deck.pot")), true);
+ // Graphviz and gettext keep the same extensions and stay accepted.
+ assert.equal(
+ await isBinaryOfficeTemplate(
+ new File(["digraph G { a -> b; }"], "graph.dot"),
+ ),
+ false,
+ );
+ assert.equal(
+ await isBinaryOfficeTemplate(
+ new File(['msgid ""\nmsgstr ""\n'], "messages.pot"),
+ ),
+ false,
+ );
+ assert.equal(await isBinaryOfficeTemplate(new File([ole], "deck.ppt")), false);
+});
+
+test("an 8-bit email is decoded with the charset it declares", async () => {
+ // A standards-valid ISO-8859-1 message is not a guess: the file says so.
+ const eml = new Uint8Array([
+ ...new TextEncoder().encode(
+ "From: [email protected]\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n" +
+ "Content-Transfer-Encoding: 8bit\r\n\r\nCaf",
+ ),
+ 0xe9,
+ 0x0a,
+ ]);
+ const text = await readTextAttachment(new File([eml], "message.eml"));
+ assert.match(text, /Caf\u00e9/);
+ assert.equal(text.includes("\uFFFD"), false);
+});
+
+test("a folded Content-Type header still yields its charset", async () => {
+ const eml = new Uint8Array([
+ ...new TextEncoder().encode(
+ "Content-Type: text/plain;\r\n\tcharset=\"ISO-8859-1\"\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ]);
+ assert.match(
+ await readTextAttachment(new File([eml], "archive.mbox")),
+ /Caf\u00e9/,
+ );
+});
+
+test("a UTF-8 email is not remapped by a stale declaration", async () => {
+ // The charset is a fallback for bytes UTF-8 rejects, never a rewrite of text
+ // that already decoded.
+ const eml = new TextEncoder().encode(
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf\u00e9 \u2014 na\u00efve",
+ );
+ assert.equal(
+ await readTextAttachment(new File([eml], "message.eml")),
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf\u00e9 \u2014 na\u00efve",
+ );
+});
+
+test("an email with no declared charset is still named, not guessed", async () => {
+ const eml = new Uint8Array([
+ ...new TextEncoder().encode("Subject: hi\r\n\r\nCaf"),
+ 0xe9,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([eml], "message.eml")),
+ (error: Error) => error instanceof UndecodableTextError,
+ );
+});
+
+test("the browser text cap matches the native one", () => {
+ // Reading happens while attaching now, so an unbounded .mbox would decode
+ // gigabytes into the webview before the user could send it.
+ assert.equal(MAX_TEXT_ATTACHMENT_BYTES, 20 * 1024 * 1024);
+ const rust = readFileSync(
+ new URL("../../src-tauri/src/native_intents.rs", import.meta.url),
+ "utf8",
+ );
+ const native = rust.match(
+ /const MAX_NATIVE_TEXT_BYTES: u64 = (\d+) \* 1024 \* 1024;/,
+ )?.[1];
+ assert.ok(native, "native text cap not found");
+ assert.equal(
+ Number(native) * 1024 * 1024,
+ MAX_TEXT_ATTACHMENT_BYTES,
+ );
+});
+
+test("a text attachment is read once, not once per stage", async () => {
+ // The composer decodes while attaching so a bad encoding is reported there;
+ // sending must reuse that answer rather than read the whole file again.
+ const file = new File(["hello"], "notes.txt");
+ let reads = 0;
+ const real = file.arrayBuffer.bind(file);
+ Object.defineProperty(file, "arrayBuffer", {
+ value: () => {
+ reads += 1;
+ return real();
+ },
+ });
+ assert.equal(await readTextAttachmentOnce(file), "hello");
+ assert.equal(await readTextAttachmentOnce(file), "hello");
+ assert.equal(reads, 1);
+ // A different file is its own entry, not a stale hit.
+ assert.equal(await readTextAttachmentOnce(new File(["bye"], "b.txt")), "bye");
+});
+
+test("both attachment stages go through the decode-once path", () => {
+ const provider = readFileSync(
new URL("../src/features/chat/runtime-provider.tsx", import.meta.url),
"utf8",
);
+ const adapter = provider.slice(
+ provider.indexOf("class TextAttachmentAdapter"),
+ provider.indexOf("class HtmlAttachmentAdapter"),
+ );
+ assert.ok(adapter.length > 0, "text adapter not found");
+ assert.equal(adapter.includes("readTextAttachmentOnce"), true);
+ // The unbounded read stays out of both stages.
+ assert.equal(/await readTextAttachment\(/.test(adapter), false);
+ // The size cap comes before any read, including the signature slices.
+ assert.ok(
+ adapter.indexOf("MAX_TEXT_ATTACHMENT_BYTES") <
+ adapter.indexOf("isBinaryPropertyList"),
+ );
+});
+
+test("every basename the picker claims is one the drop paths accept", () => {
+ // assistant-ui derives the extension as `.${name.split(".").pop()}`, so an
+ // extensionless "Dockerfile" matches the ".dockerfile" token and the picker
+ // takes it. A drop of the same file has to agree, or the two disagree on one
+ // conventional build file.
+ for (const name of ["Dockerfile", "Makefile", "Containerfile"]) {
+ const derived = `.${name.split(".").pop()!.toLowerCase()}`;
+ assert.ok(
+ TEXT_ATTACHMENT_EXTENSIONS.includes(derived),
+ `${name} is claimed through ${derived}`,
+ );
+ assert.equal(isTextAttachmentName(name), true, `${name} is droppable`);
+ }
+ // A name with no matching token stays out of both, rather than being widened.
+ assert.equal(TEXT_ATTACHMENT_EXTENSIONS.includes(".gnumakefile"), false);
+ assert.equal(isTextAttachmentName("GNUmakefile"), false);
+});
+
+test("a container declaring two charsets is refused, not part-decoded", async () => {
+ // Decoding the whole file with the first declaration would corrupt every
+ // later part, which is the failure mode the strict decoder exists to avoid.
+ const mixed = new Uint8Array([
+ ...new TextEncoder().encode(
+ "Content-Type: multipart/mixed; boundary=b\r\n\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...new TextEncoder().encode(
+ "\r\n--b\r\nContent-Type: text/plain; charset=windows-1251\r\n\r\n",
+ ),
+ 0xcf,
+ 0xf0,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([mixed], "thread.mbox")),
+ (error: Error) => {
+ assert.ok(error instanceof UndecodableTextError);
+ assert.match(error.message, /declares more than one charset/);
+ return true;
+ },
+ );
+});
+
+test("a multipart message whose parts agree is decoded with that charset", async () => {
+ // The outer header carries no charset, so reading only the first Content-Type
+ // rejected a file that says plainly what it is.
+ const eml = new Uint8Array([
+ ...new TextEncoder().encode(
+ "Content-Type: multipart/mixed; boundary=b\r\n\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ]);
assert.match(
- src,
- /class TextAttachmentAdapter implements AttachmentAdapter \{[\s\S]*?accept = TEXT_ATTACHMENT_ACCEPT;/,
+ await readTextAttachment(new File([eml], "message.eml")),
+ /Caf\u00e9/,
);
- for (const ext of [".cs", ".php", ".js"]) {
- assert.ok(TEXT_ATTACHMENT_ACCEPT.includes(ext), ext);
+});
+
+test("a vCard property charset is honoured like an email's", async () => {
+ // vCard 2.1 puts the encoding on the property rather than in a header.
+ const vcf = new Uint8Array([
+ ...new TextEncoder().encode(
+ "BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf",
+ ),
+ 0xe9,
+ ...new TextEncoder().encode("\r\nEND:VCARD\r\n"),
+ ]);
+ assert.match(
+ await readTextAttachment(new File([vcf], "contact.vcf")),
+ /Caf\u00e9/,
+ );
+});
+
+test("compare mode takes the same audio files the chat composer does", () => {
+ // Browsers leave file.type empty for several of these containers, so a
+ // MIME-only check dropped them silently.
+ for (const name of ["clip.wma", "voice.amr", "note.caf", "take.aiff"]) {
+ assert.equal(isAudioAttachmentFile(new File([], name, { type: "" })), true);
}
+ assert.equal(
+ isAudioAttachmentFile(new File([], "song.mp3", { type: "audio/mpeg" })),
+ true,
+ );
+ assert.equal(isAudioAttachmentFile(new File([], "notes.txt", { type: "" })), false);
+ // The compare composer classifies through the shared helper, not file.type.
+ const composer = readFileSync(
+ new URL("../src/features/chat/shared-composer.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.equal(/file\.type\.match\(\/\^audio/.test(composer), false);
+ assert.match(composer, /isAudioAttachmentFile\(file\)/);
+ assert.match(composer, /accept=\{AUDIO_PICKER_ACCEPT\}/);
+});
- const attachmentContentSource = readFileSync(
- new URL("../src/features/chat/attachment-content.ts", import.meta.url),
+test("a declaration past the first pages is not missed", async () => {
+ // A prefix-scan cutoff decoded the whole archive as the first message's
+ // charset, so a later message in another one came out as mojibake. Any cutoff
+ // has that failure, which is why the scan reads the file the cap already
+ // bounds. Padded well past the 64 KiB the scan used to stop at.
+ const enc = new TextEncoder();
+ const mbox = new Uint8Array([
+ ...enc.encode(
+ "From [email protected] Mon Jan 1 00:00:00 2024\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\n" +
+ "x".repeat(70 * 1024) +
+ "\r\n\r\nFrom [email protected] Mon Jan 1 00:00:00 2024\r\n" +
+ "Content-Type: text/plain; charset=windows-1251\r\n\r\n",
+ ),
+ 0xcf,
+ 0xf0,
+ 0xe8,
+ 0xe2,
+ 0xe5,
+ 0xf2,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([mbox], "inbox.mbox")),
+ (error: Error) => {
+ assert.ok(error instanceof UndecodableTextError);
+ assert.match(error.message, /ISO-8859-1, windows-1251/);
+ return true;
+ },
+ );
+ // The source carries no byte ceiling on the scan for a cutoff to creep back in.
+ const source = readFileSync(
+ new URL(
+ "../src/features/chat/text-attachment-accept.ts",
+ import.meta.url,
+ ),
"utf8",
);
+ assert.equal(source.includes("DECLARATION_SCAN_BYTES"), false);
+});
+
+test("a header-shaped line in the body is not a declaration", async () => {
+ // Quoted mail and config snippets put "Content-Type:" at the start of a body
+ // line. Counting those as declarations refused files that say one thing.
+ const eml = new Uint8Array([
+ ...new TextEncoder().encode(
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\n" +
+ "Here is the config we discussed:\r\n" +
+ "Content-Type: text/html; charset=windows-1251\r\n" +
+ "and that is the end of it. Caf",
+ ),
+ 0xe9,
+ ]);
assert.match(
- attachmentContentSource,
- /import \{ TEXT_ATTACHMENT_ACCEPT \} from "\.\/text-attachment-accept";/,
+ await readTextAttachment(new File([eml], "message.eml")),
+ /Caf\u00e9/,
+ );
+});
+
+test("parts after a boundary are still read as headers", async () => {
+ // The narrowing must not lose real part headers, which is what made the
+ // multipart case work in the first place.
+ const enc = new TextEncoder();
+ const mixed = new Uint8Array([
+ ...enc.encode(
+ "Content-Type: multipart/mixed; boundary=b\r\n\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...enc.encode("\r\n--b\r\nContent-Type: text/plain; charset=windows-1251\r\n\r\n"),
+ 0xcf,
+ 0xf0,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([mixed], "message.eml")),
+ (error: Error) => {
+ assert.match(error.message, /ISO-8859-1, windows-1251/);
+ return true;
+ },
);
});
+
+test("a signature marker is not a MIME boundary", async () => {
+ // "-- " opens a signature block in most mail clients. Treating every line
+ // starting with "--" as a boundary put the body back into header mode, so a
+ // quoted header below it became a second declaration and refused the file.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode("Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf"),
+ 0xe9,
+ ...enc.encode(
+ "\r\n-- \r\nSent from a phone\r\n" +
+ "Content-Type: text/plain; charset=windows-1251\r\n",
+ ),
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "message.eml")), /Caf\u00e9/);
+});
+
+test("a closing delimiter ends its part instead of starting one", async () => {
+ // "--b--" closes the multipart; the epilogue after it is body text, so a
+ // header-shaped line there declares nothing.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode(
+ "Content-Type: multipart/mixed; boundary=b\r\n\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...enc.encode("\r\n--b--\r\nContent-Type: text/plain; charset=windows-1251\r\n"),
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "thread.mbox")), /Caf\u00e9/);
+});
+
+test("a vCard charset in a value is not a property parameter", async () => {
+ // The escaped semicolon is part of the NOTE text. Matching it added a second
+ // declaration and refused a card that names exactly one charset.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nNOTE:Write \\;CHARSET=windows-1251 to pin it\r\nEND:VCARD\r\n"),
+ ]);
+ assert.match(await readTextAttachment(new File([vcf], "contact.vcf")), /Caf\u00e9/);
+});
+
+test("each vCard property is read under its own declared charset", async () => {
+ // CHARSET is a property parameter, so a card naming two of them says plainly
+ // what each value holds. Reading it as one unit corrupted all but one.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nORG;CHARSET=windows-1251:"),
+ 0xcf,
+ 0xf0,
+ ...enc.encode("\r\nTEL:+15551234\r\nEND:VCARD\r\n"),
+ ]);
+ const text = await readTextAttachment(new File([vcf], "contact.vcf"));
+ assert.match(text, /FN;CHARSET=windows-1252:Café/);
+ assert.match(text, /ORG;CHARSET=windows-1251:Пр/);
+ // Names, parameters, undeclared properties and line endings survive intact.
+ assert.match(text, /TEL:\+15551234/);
+ assert.ok(text.startsWith("BEGIN:VCARD\r\nVERSION:2.1\r\n"));
+ assert.ok(text.endsWith("END:VCARD\r\n"));
+});
+
+test("a folded value keeps the charset of the property above it", async () => {
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nNOTE;CHARSET=windows-1251:"),
+ 0xcf,
+ ...enc.encode("\r\n "),
+ 0xf0,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ const text = await readTextAttachment(new File([vcf], "contact.vcf"));
+ assert.match(text, /NOTE;CHARSET=windows-1251:П\r\n р/);
+});
+
+test("a vCard naming a charset it does not hold is still refused", async () => {
+ // The per-property read only replaces the refusal when it can account for
+ // every value; a property whose bytes break its own declaration is not that.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=utf-8:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nORG;CHARSET=windows-1251:"),
+ 0xcf,
+ 0xf0,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([vcf], "contact.vcf")),
+ (error: Error) => {
+ // The property that broke its declaration, not a list of every charset
+ // the card names: only one of them is the reason it cannot be read.
+ assert.match(
+ error.message,
+ /declares charset "utf-8" but does not hold valid utf-8 text/,
+ );
+ return true;
+ },
+ );
+});
+
+test("a multipart mail declaring two charsets is still refused", async () => {
+ // Only the vCard reading changed: a MIME container needs a parser this is not.
+ const enc = new TextEncoder();
+ const mixed = new Uint8Array([
+ ...enc.encode(
+ "Content-Type: multipart/mixed; boundary=b\r\n\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...enc.encode("\r\n--b\r\nContent-Type: text/plain; charset=windows-1251\r\n\r\n"),
+ 0xcf,
+ 0xf0,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([mixed], "message.eml")),
+ (error: Error) => {
+ assert.match(error.message, /declares more than one charset/);
+ return true;
+ },
+ );
+});
+
+test("an XML prolog encoding is honoured for every XML dialect", async () => {
+ // .resx, .xliff and friends are XML documents that state their encoding, so
+ // refusing them for carrying the bytes they describe was wrong.
+ const enc = new TextEncoder();
+ for (const name of ["Strings.resx", "ui.xliff", "app.xlf", "icon.svg"]) {
+ const bytes = new Uint8Array([
+ ...enc.encode('<?xml version="1.0" encoding="windows-1252"?>\n<root>Caf'),
+ 0xe9,
+ ...enc.encode("</root>\n"),
+ ]);
+ assert.match(
+ await readTextAttachment(new File([bytes], name)),
+ /Caf\u00e9/,
+ `${name} decodes under its declared encoding`,
+ );
+ }
+ // A prolog only counts at the start of the document, where the spec puts it.
+ const late = new Uint8Array([
+ ...enc.encode('# notes\n<?xml version="1.0" encoding="windows-1252"?>\nCaf'),
+ 0xe9,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([late], "notes.txt")),
+ (error: Error) => error instanceof UndecodableTextError,
+ );
+});
+
+test("the tracker magic tables agree, and cover ProTracker's second marker", async () => {
+ // The two lists are hand-mirrored, so drift is only visible if something
+ // compares them. !PM! is the marker that was missing from both: file(1) lists
+ // it at 1080 beside M.K., and a 31-sample module puts byte 470 inside a
+ // sample name rather than the order table, so the Soundtracker fallback does
+ // not catch one either and the module was read as UTF-8 text.
+ const rust = readFileSync(
+ new URL("../../src-tauri/src/native_path_policy.rs", import.meta.url),
+ "utf8",
+ );
+ const table = rust.match(
+ /const TRACKER_MOD_MAGICS: &\[&\[u8; 4\]\] = &\[([\s\S]*?)\];/,
+ );
+ assert.ok(table, "TRACKER_MOD_MAGICS not found in native_path_policy.rs");
+ const rustMagics = [...table[1]!.matchAll(/b"((?:[^"\\]|\\.){1,8})"/g)].map((m) =>
+ m[1]!.replace(/\\0/g, "\0"),
+ );
+ assert.ok(rustMagics.includes("!PM!"), "Rust table is missing !PM!");
+
+ const module = await import("../src/features/chat/text-attachment-accept.ts");
+ const source = readFileSync(
+ new URL("../src/features/chat/text-attachment-accept.ts", import.meta.url),
+ "utf8",
+ );
+ const tsTable = source.match(/const TRACKER_MOD_MAGICS = new Set\(\[([\s\S]*?)\]\)/);
+ assert.ok(tsTable, "TRACKER_MOD_MAGICS not found in text-attachment-accept.ts");
+ const tsMagics = [...tsTable[1]!.matchAll(/"((?:[^"\\]|\\.){1,8})"/g)].map((m) =>
+ m[1]!.replace(/\\0/g, "\0"),
+ );
+ assert.deepEqual([...tsMagics].sort(), [...rustMagics].sort());
+ assert.ok(module.isBinaryTrackerModule, "the sniffer is still exported");
+
+ // And the marker is actually acted on, not merely listed.
+ const bytes = new Uint8Array(1084 + 4);
+ bytes.set(new TextEncoder().encode("!PM!"), 1080);
+ assert.equal(
+ await module.isBinaryTrackerModule(new File([bytes], "tune.mod")),
+ true,
+ );
+ // A go.mod of the same length is still text.
+ const text = new Uint8Array(1084 + 4).fill(0x20);
+ assert.equal(
+ await module.isBinaryTrackerModule(new File([text], "go.mod")),
+ false,
+ );
+});
+
+test("a charset with no decoder here is reported, not swallowed", async () => {
+ // The composer only toasts an UndecodableTextError; a bare Error failed that
+ // check and the attachment vanished with no message at all. These are not
+ // exotic labels: the Encoding Standard maps the ISO-2022-KR, ISO-2022-CN and
+ // HZ-GB-2312 families to "replacement", which TextDecoder is required to
+ // refuse, so a Korean or Chinese card declaring its own charset went out
+ // silently on every engine.
+ for (const charset of ["ISO-2022-KR", "HZ-GB-2312", "ISO-2022-CN", "cp437"]) {
+ const card = new File(
+ [`BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=${charset}:name\r\nEND:VCARD\r\n`],
+ "contact.vcf",
+ );
+ await assert.rejects(
+ readTextAttachment(card),
+ (error: Error) => {
+ assert.ok(
+ error instanceof UndecodableTextError,
+ `${charset} produced a ${error.name}, which the composer does not toast`,
+ );
+ assert.match(error.message, new RegExp(charset, "i"));
+ return true;
+ },
+ charset,
+ );
+ }
+});
+
+test("a declared charset is decoded as strictly as an undeclared one", async () => {
+ // The declared-charset paths used a lenient decoder, so a file claiming UTF-8
+ // and holding broken bytes came through as replacement characters, which is
+ // the corruption the strict default was added to stop.
+ const enc = new TextEncoder();
+ const broken = [0xc3, 0x28]; // a lead byte followed by an invalid continuation
+ for (const [name, head] of [
+ [
+ "messages.po",
+ 'msgid ""\nmsgstr ""\n"Content-Type: text/plain; charset=UTF-8\\n"\n\nmsgstr "Caf',
+ ],
+ ["strings.resx", '<?xml version="1.0" encoding="UTF-8"?><r>Caf'],
+ ] as const) {
+ await assert.rejects(
+ readTextAttachment(new File([new Uint8Array([...enc.encode(head), ...broken])], name)),
+ (error: Error) => {
+ assert.ok(error instanceof UndecodableTextError);
+ assert.match(error.message, /declares charset "UTF-8"/);
+ return true;
+ },
+ `${name} is refused rather than corrupted`,
+ );
+ }
+ // A single-byte charset maps every byte, so those files are unaffected.
+ const latin = new Uint8Array([
+ ...enc.encode('<?xml version="1.0" encoding="windows-1252"?><r>Caf'),
+ 0xe9,
+ ]);
+ assert.match(
+ await readTextAttachment(new File([latin], "strings.resx")),
+ /Caf\u00e9/,
+ );
+ // A bounded preview keeps its allowance for the character the slice cut.
+ const cut = new Uint8Array([
+ ...enc.encode('<?xml version="1.0" encoding="UTF-8"?><r>caf'),
+ 0xc3,
+ ]);
+ assert.match(
+ decodeTextAttachmentBytes(cut, "strings.resx", true),
+ /<r>caf$/,
+ );
+});
+
+test("an XML prolog decides the encoding before UTF-8 is tried", async () => {
+ // 0xC3 0xA9 is valid UTF-8 for "e-acute", so a UTF-8-first decode succeeded and
+ // the prolog was never consulted. Per the spec those bytes are two windows-1252
+ // characters, which is what an XML parser reads and what we must send.
+ const bytes = new Uint8Array([
+ ...new TextEncoder().encode('<?xml version="1.0" encoding="windows-1252"?><r>'),
+ 0xc3,
+ 0xa9,
+ ...new TextEncoder().encode("</r>"),
+ ]);
+ const text = await readTextAttachment(new File([bytes], "strings.resx"));
+ assert.match(text, /\u00c3\u00a9/, "decoded as the prolog says, not as UTF-8");
+});
+
+test("a BOM-marked file decodes as strictly as everything else", async () => {
+ // An odd trailing byte used to be padded with a replacement character and
+ // attached as though it had been read.
+ const odd = new Uint8Array([0xff, 0xfe, 0x61, 0x00, 0x62]);
+ await assert.rejects(
+ readTextAttachment(new File([odd], "export.reg")),
+ (error: Error) => error instanceof UndecodableTextError,
+ );
+ // A bounded preview still drops only the incomplete unit at the cut.
+ assert.equal(decodeTextAttachmentBytes(odd, "export.reg", true), "a");
+ // Well-formed UTF-16 is unaffected.
+ const good = new Uint8Array([0xff, 0xfe, 0x61, 0x00, 0x62, 0x00]);
+ assert.equal(await readTextAttachment(new File([good], "export.reg")), "ab");
+});
+
+test("a body of boundary-shaped lines does not stall the composer", () => {
+ // Restarting the header search from every candidate boundary was quadratic:
+ // a megabyte of diff hunks took 14 seconds, inside add(), on the UI thread.
+ const body = "--- a/file.txt\n".repeat(70_000);
+ const bytes = new Uint8Array([
+ ...new TextEncoder().encode(
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\n" + body,
+ ),
+ 0xe9,
+ ]);
+ const started = performance.now();
+ decodeTextAttachmentBytes(bytes, "thread.mbox");
+ assert.ok(
+ performance.now() - started < 2_000,
+ "the scan is linear in the file, not quadratic in its boundary-shaped lines",
+ );
+});
+
+test("legacy M3U playlists are not advertised as UTF-8 text", () => {
+ assert.equal(TEXT_ATTACHMENT_EXTENSIONS.includes(".m3u"), false);
+ assert.equal(TEXT_ATTACHMENT_EXTENSIONS.includes(".m3u8"), true);
+});
+
+test("UTF-16 registry exports are decoded before attachment", async () => {
+ const text =
+ "Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Test]";
+ const utf16le = new Uint8Array(2 + text.length * 2);
+ utf16le.set([0xff, 0xfe]);
+ for (let index = 0; index < text.length; index += 1) {
+ const codeUnit = text.charCodeAt(index);
+ utf16le[2 + index * 2] = codeUnit & 0xff;
+ utf16le[3 + index * 2] = codeUnit >>> 8;
+ }
+
+ assert.equal(
+ await readTextAttachment(new File([utf16le], "export.reg")),
+ text,
+ );
+ assert.equal(
+ await readTextAttachment(new File(["plain UTF-8"], "notes.txt")),
+ "plain UTF-8",
+ );
+});
+
+test("gettext catalogs use the charset declared by their header", async () => {
+ const before =
+ 'msgid ""\nmsgstr ""\n"Content-Type: text/plain; charset=ISO-8859-1\\n"\n\nmsgid "coffee"\nmsgstr "caf';
+ const after = '"\n';
+ const prefix = new TextEncoder().encode(before);
+ const suffix = new TextEncoder().encode(after);
+ const encoded = new Uint8Array(prefix.length + 1 + suffix.length);
+ encoded.set(prefix);
+ encoded[prefix.length] = 0xe9;
+ encoded.set(suffix, prefix.length + 1);
+
+ assert.equal(
+ await readTextAttachment(new File([encoded], "messages.po")),
+ `${before}é${after}`,
+ );
+});
+
+test("RAG documents keep their existing route", () => {
+ for (const path of ["/docs/notes.txt", "/docs/notes.md", "/docs/paper.pdf"]) {
+ assert.equal(classifyDropPaths([path]).kind, "docs", path);
+ assert.equal(isComposerAttachmentName(path), false, path);
+ }
+});
+
+test("a dotfile is not mistaken for a source file", () => {
+ // Rust classifies on Path::extension, which ".env" does not have.
+ for (const path of ["/p/.env", "/p/.properties", "/p/.log"]) {
+ assert.equal(isComposerAttachmentName(path), false, path);
+ assert.equal(classifyDropPaths([path]).kind, "unsupported", path);
+ }
+ assert.equal(isComposerAttachmentName("/p/app.env"), true);
+});
+
+test("an unreadable type is still refused", () => {
+ assert.equal(classifyDropPaths(["/docs/archive.zip"]).kind, "unsupported");
+ assert.equal(classifyDropPaths(["/bin/tool.exe"]).kind, "unsupported");
+});
+
+test("frontend and Rust accept the same dropped text extensions", () => {
+ const docs = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>
+ ext.trim().toLowerCase(),
+ );
+ const frontend = TEXT_ATTACHMENT_EXTENSIONS.map((ext) => ext.toLowerCase())
+ .filter((ext) => !docs.includes(ext))
+ .sort();
+ const rustSource = readFileSync(
+ new URL("../../src-tauri/src/native_path_policy.rs", import.meta.url),
+ "utf8",
+ );
+ const rust = [
+ ...(rustSource
+ .match(RUST_TEXT_ATTACHMENT_EXTS_RE)?.[1]
+ .matchAll(RUST_EXTENSION_RE) ?? []),
+ ]
+ .map((match) => `.${match[1]}`)
+ .sort();
+
+ assert.deepEqual(rust, frontend);
+});
+
+test("the composer adapter reads the shared text accept list", () => {
+ const src = readFileSync(
+ new URL("../src/features/chat/runtime-provider.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(
+ src,
+ /class TextAttachmentAdapter implements AttachmentAdapter \{[\s\S]*?accept = TEXT_ATTACHMENT_ACCEPT;/,
+ );
+ assert.match(src, /if \(await isBinaryTrackerModule\(file\)\)/);
+ for (const ext of [".cs", ".php", ".js"]) {
+ assert.ok(TEXT_ATTACHMENT_ACCEPT.includes(ext), ext);
+ }
+
+ const attachmentContentSource = readFileSync(
+ new URL("../src/features/chat/attachment-content.ts", import.meta.url),
+ "utf8",
+ );
+ assert.match(
+ attachmentContentSource,
+ /import \{[\s\S]*?TEXT_ATTACHMENT_ACCEPT[\s\S]*?decodeTextAttachmentBytes[\s\S]*?\} from "\.\/text-attachment-accept";/,
+ );
+});
+
+// Mirrors `bmff_box` and `three_gp_with_tracks` in native_intents.rs, so the
+// browser classifier is tested against the same containers as the native one.
+// The return types name their buffer: a bare Uint8Array is generic over
+// ArrayBufferLike, which includes SharedArrayBuffer, and BlobPart takes neither
+// that nor a view onto it. Every one of these ends up inside a File.
+function bmffBox(kind: string, payload: Uint8Array): Uint8Array<ArrayBuffer> {
+ const boxed = new Uint8Array(8 + payload.length);
+ new DataView(boxed.buffer).setUint32(0, boxed.length);
+ boxed.set(new TextEncoder().encode(kind), 4);
+ boxed.set(payload, 8);
+ return boxed;
+}
+
+function threeGpWithTracks(handlers: readonly string[]): Uint8Array<ArrayBuffer> {
+ const traks: Uint8Array<ArrayBuffer>[] = [];
+ for (const handler of handlers) {
+ const hdlrPayload = new Uint8Array(12);
+ hdlrPayload.set(new TextEncoder().encode(handler), 8);
+ traks.push(bmffBox("trak", bmffBox("mdia", bmffBox("hdlr", hdlrPayload))));
+ }
+ const moovPayload = new Uint8Array(
+ traks.reduce((total, trak) => total + trak.length, 0),
+ );
+ let offset = 0;
+ for (const trak of traks) {
+ moovPayload.set(trak, offset);
+ offset += trak.length;
+ }
+ return bmffBox("moov", moovPayload);
+}
+
+test("3GP tracks decide audio or video, as they do natively", () => {
+ assert.equal(isAudioOnly3gpBytes(threeGpWithTracks(["soun"])), true);
+ assert.equal(isAudioOnly3gpBytes(threeGpWithTracks(["vide"])), false);
+ assert.equal(isAudioOnly3gpBytes(threeGpWithTracks(["soun", "vide"])), false);
+ // Not a container at all, and a truncated one.
+ assert.equal(isAudioOnly3gpBytes(new Uint8Array([1, 2, 3])), false);
+ assert.equal(
+ isAudioOnly3gpBytes(threeGpWithTracks(["soun"]).subarray(0, 20)),
+ false,
+ );
+});
+
+test("an audio-only 3GP recording is classified as audio, not video", async () => {
+ // The browser answers "" or video/3gpp for both kinds, and the video accept
+ // list claims .3gp, so without this the recording reached the video adapter.
+ const recording = new File([threeGpWithTracks(["soun"])], "voice.3gp", {
+ type: "",
+ });
+ const classified = await classifiedAttachmentFile(recording);
+ assert.equal(classified.type, "audio/3gpp");
+ assert.equal(classified.name, "voice.3gp");
+ assert.equal(isAudioAttachmentFile(classified), true);
+});
+
+test("a real 3GP clip stays video", async () => {
+ const clip = new File([threeGpWithTracks(["soun", "vide"])], "clip.3gp", {
+ type: "video/3gpp",
+ });
+ assert.equal(await classifiedAttachmentFile(clip), clip);
+});
+
+test("classification leaves every other container untouched", async () => {
+ for (const [name, type] of [
+ ["clip.mp4", "video/mp4"],
+ ["take.mkv", ""],
+ ["voice.m4a", ""],
+ ["notes.txt", "text/plain"],
+ ] as const) {
+ const file = new File([new Uint8Array([0, 1, 2, 3])], name, { type });
+ assert.equal(await classifiedAttachmentFile(file), file, name);
+ }
+});
+
+test("the restamped recording routes to the audio adapter", async () => {
+ // The composite's own matcher, not a copy of it: audio is registered before
+ // video, so a file the audio accept list claims never reaches the video one.
+ const { fileMatchesAccept } = (await import(
+ new URL(
+ "../node_modules/@assistant-ui/core/dist/adapters/attachment.js",
+ import.meta.url,
+ ).href
+ )) as { fileMatchesAccept: (file: File, accept: string) => boolean };
+ const recording = new File([threeGpWithTracks(["soun"])], "voice.3gp", {
+ type: "",
+ });
+ assert.equal(fileMatchesAccept(recording, AUDIO_ATTACHMENT_ACCEPT), false);
+ assert.equal(fileMatchesAccept(recording, VIDEO_ACCEPT), true);
+
+ const classified = await classifiedAttachmentFile(recording);
+ assert.equal(fileMatchesAccept(classified, AUDIO_ATTACHMENT_ACCEPT), true);
+});
+
+test("a file dialog offers 3GP recordings, and routing still does not", () => {
+ // A dialog decides what is selectable, so it has to name .3gp: a platform
+ // that maps it to video/3gpp, or to nothing, greys the recording out. Routing
+ // must not, or the audio adapter would claim every 3GP video ahead of the
+ // video one, which is matched after it.
+ assert.ok(AUDIO_PICKER_ACCEPT.split(",").includes(".3gp"));
+ assert.equal(AUDIO_ATTACHMENT_ACCEPT.split(",").includes(".3gp"), false);
+ assert.ok(AUDIO_PICKER_ACCEPT.startsWith(AUDIO_ATTACHMENT_ACCEPT));
+
+ const adapter = readFileSync(
+ new URL(
+ "../src/features/chat/audio-attachment-adapter.ts",
+ import.meta.url,
+ ),
+ "utf8",
+ );
+ assert.match(adapter, /accept = AUDIO_ATTACHMENT_ACCEPT;/);
+ assert.equal(/AUDIO_PICKER_ACCEPT/.test(adapter), false);
+});
+
+test("a 3GP clip picked through the audio dialog is still refused as video", async () => {
+ const clip = new File([threeGpWithTracks(["soun", "vide"])], "clip.3gp", {
+ type: "video/3gpp",
+ });
+ const [classified] = await classifiedAttachmentFiles([clip]);
+ assert.equal(isAudioAttachmentFile(classified!), false);
+ assert.equal(isVideoFile(classified!), true);
+});
+
+test("the composer classifies before an adapter is picked", () => {
+ // A composite matches on name and MIME synchronously, so the restamping has
+ // to happen in the wrapper above it rather than inside an adapter.
+ const src = readFileSync(
+ new URL("../src/features/chat/runtime-provider.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(
+ src,
+ /class PreStreamAwareAttachmentAdapter[\s\S]*?if \(!needsAttachmentTrackInspection\(state\.file\)\)[\s\S]*?await classifiedAttachmentFile\(state\.file\)/,
+ );
+ const composer = readFileSync(
+ new URL("../src/features/chat/shared-composer.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(composer, /await classifiedAttachmentFiles\(input\)/);
+});
+
+test("a From line separates messages in an archive, not in one message", async () => {
+ // An mbox escapes a body line that starts with "From "; a standalone .eml has
+ // no separator at all, so an ordinary sentence opening that way is body text.
+ const enc = new TextEncoder();
+ const bytes = new Uint8Array([
+ ...enc.encode("Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf"),
+ 0xe9,
+ ...enc.encode(
+ "\r\nFrom here on it quotes a header:\r\n" +
+ "Content-Type: text/plain; charset=windows-1251\r\n",
+ ),
+ ]);
+ assert.match(
+ await readTextAttachment(new File([bytes], "message.eml")),
+ /Café/,
+ );
+ await assert.rejects(
+ readTextAttachment(new File([bytes], "thread.mbox")),
+ (error: Error) => {
+ assert.match(error.message, /ISO-8859-1, windows-1251/);
+ return true;
+ },
+ );
+});
+
+function threeGpFile(
+ name: string,
+ handlers: readonly string[],
+ mdatBytes: number,
+): File {
+ const ftyp = bmffBox("ftyp", new TextEncoder().encode("3gp4isom"));
+ const moov = threeGpWithTracks(handlers);
+ const mdat = bmffBox("mdat", new Uint8Array(mdatBytes));
+ return new File([ftyp, moov, mdat], name, { type: "" });
+}
+
+/** Records every range read from a file, and forbids reading it whole. */
+function watchReads(file: File): Array<[number, number]> {
+ const reads: Array<[number, number]> = [];
+ const slice = file.slice.bind(file);
+ Object.defineProperty(file, "slice", {
+ value: (start: number, end: number) => {
+ reads.push([start, end]);
+ return slice(start, end);
+ },
+ });
+ Object.defineProperty(file, "arrayBuffer", {
+ value: () => {
+ throw new Error("the whole container must not be read");
+ },
+ });
+ return reads;
+}
+
+test("only the track table is read, not the samples beside it", async () => {
+ // moov holds the handlers and mdat the audio, so reading the file to reach a
+ // handler retains the whole clip: 64 MB for one, and that per file in a drop.
+ const file = threeGpFile("voice.3gp", ["soun"], 2 * 1024 * 1024);
+ const reads = watchReads(file);
+
+ const classified = await classifiedAttachmentFile(file);
+ assert.equal(classified.type, "audio/3gpp");
+ const bytesRead = reads.reduce((total, [start, end]) => total + end - start, 0);
+ assert.ok(
+ bytesRead < 4096,
+ `read ${bytesRead} bytes of a ${file.size}-byte container`,
+ );
+});
+
+test("a dropped batch is inspected one container at a time", async () => {
+ // Promise.all put every container's payload in memory at once, so ten clips
+ // at the 64 MB limit came to 640 MB before anything could be rejected.
+ const files = ["a.3gp", "b.3gp", "c.3gp"].map((name) =>
+ threeGpFile(name, ["soun"], 64 * 1024),
+ );
+ const order: string[] = [];
+ for (const file of files) {
+ const slice = file.slice.bind(file);
+ Object.defineProperty(file, "slice", {
+ value: (start: number, end: number) => {
+ order.push(file.name);
+ return slice(start, end);
+ },
+ });
+ }
+
+ const classified = await classifiedAttachmentFiles(files);
+ assert.deepEqual(
+ classified.map((file) => file.type),
+ ["audio/3gpp", "audio/3gpp", "audio/3gpp"],
+ );
+ // Grouped, not interleaved: every read of a file precedes the next one's.
+ assert.deepEqual([...new Set(order)], ["a.3gp", "b.3gp", "c.3gp"]);
+ assert.deepEqual(
+ order,
+ [...order].sort((left, right) => left.localeCompare(right)),
+ );
+});
+
+test("a vCard charset decides before UTF-8, like the other declarations", async () => {
+ // C3 A9 is valid UTF-8 for "é" and two windows-1252 characters. The card says
+ // which it holds, so decoding as UTF-8 delivered characters it never named.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xc3,
+ 0xa9,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ assert.match(
+ await readTextAttachment(new File([vcf], "contact.vcf")),
+ /Café/,
+ );
+ // A card that names UTF-8 still reads as UTF-8, so nothing regresses there.
+ const utf8 = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:3.0\r\nFN;CHARSET=UTF-8:Caf"),
+ 0xc3,
+ 0xa9,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ assert.match(await readTextAttachment(new File([utf8], "modern.vcf")), /Café/);
+ // A card with no charset parameter at all is untouched by this.
+ const plain = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:4.0\r\nFN:Caf"),
+ 0xc3,
+ 0xa9,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ assert.match(await readTextAttachment(new File([plain], "v4.vcf")), /Café/);
+});
+
+test("a mail Content-Type stays a fallback, unlike the vCard parameter", async () => {
+ // Clients mislabel 8-bit mail constantly, so valid UTF-8 outranks the header.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode("Content-Type: text/plain; charset=windows-1252\r\n\r\nCaf"),
+ 0xc3,
+ 0xa9,
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "message.eml")), /Café/);
+});
+
+test("a boundary does not carry into the next message of an archive", async () => {
+ // Each message declares its own. A later body line repeating an earlier one
+ // reopened the headers there, and the next header-shaped line counted.
+ const enc = new TextEncoder();
+ const mbox = new Uint8Array([
+ ...enc.encode(
+ "From [email protected] Thu Aug 27 00:00:00 2026\r\n" +
+ "Content-Type: multipart/mixed; boundary=b\r\n\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...enc.encode(
+ "\r\nFrom [email protected] Thu Aug 27 01:00:00 2026\r\n" +
+ "Subject: a plain message\r\n\r\n" +
+ "The separator line below is body text here:\r\n--b\r\n" +
+ "Content-Type: text/plain; charset=windows-1251\r\n",
+ ),
+ ]);
+ assert.match(await readTextAttachment(new File([mbox], "thread.mbox")), /Café/);
+});
+
+test("a gettext header past the prefix is still read", async () => {
+ // A catalog with more than 64 KiB of translator comments above its header was
+ // refused for carrying exactly the bytes that header describes.
+ const enc = new TextEncoder();
+ const comments = "# a translator comment line\n".repeat(4_000);
+ assert.ok(comments.length > 64 * 1024);
+ const po = new Uint8Array([
+ ...enc.encode(
+ `${comments}msgid ""\nmsgstr ""\n` +
+ '"Content-Type: text/plain; charset=windows-1251\\n"\n\n' +
+ 'msgid "hi"\nmsgstr "',
+ ),
+ 0xcf,
+ 0xf0,
+ ...enc.encode('"\n'),
+ ]);
+ assert.match(await readTextAttachment(new File([po], "ru.po")), /Пр/);
+});
+
+test("the reference picker takes the formats its drop path does", async () => {
+ const { REFERENCE_PICKER_ACCEPT, referenceFileRejection } = await import(
+ "../src/features/video/reference-budget.ts"
+ );
+ // A browser answers "" for these, so `audio/*` alone hid and then refused them.
+ for (const name of ["voice.amr", "clip.wma", "note.caf", "take.aiff"]) {
+ assert.equal(
+ referenceFileRejection("audio", { type: "", size: 10, name }),
+ null,
+ name,
+ );
+ assert.ok(REFERENCE_PICKER_ACCEPT.audio.includes(name.slice(name.indexOf("."))));
+ }
+ assert.equal(
+ referenceFileRejection("video", { type: "", size: 10, name: "take.mkv" }),
+ null,
+ );
+ // The kind check still holds, by name as well as by type.
+ assert.equal(
+ referenceFileRejection("audio", { type: "", size: 10, name: "clip.mkv" }),
+ "Please choose an audio file",
+ );
+ assert.equal(
+ referenceFileRejection("video", { type: "image/png", size: 10 }),
+ "Please choose a video file",
+ );
+ // Every extension the native drop accepts is offered by the dialog too.
+ for (const ext of CHAT_AUDIO_DROP_ACCEPT.split(",")) {
+ assert.ok(REFERENCE_PICKER_ACCEPT.audio.split(",").includes(ext), ext);
+ }
+ for (const ext of CHAT_VIDEO_DROP_ACCEPT.split(",")) {
+ assert.ok(REFERENCE_PICKER_ACCEPT.video.split(",").includes(ext), ext);
+ }
+
+ const picker = readFileSync(
+ new URL("../src/features/video/reference-picker.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(picker, /accept=\{REFERENCE_PICKER_ACCEPT\[kind\]\}/);
+});
+
+test("one vCard declaration speaks for its property, not for the file", async () => {
+ // A 2.1 card beside a 3.0 one, which declares nothing because it cannot. The
+ // whole-file reading turned the 3.0 record's UTF-8 into windows-1252 mojibake.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nEND:VCARD\r\nBEGIN:VCARD\r\nVERSION:3.0\r\nFN:Stra"),
+ 0xc3,
+ 0x9f,
+ ...enc.encode("e\r\nEND:VCARD\r\n"),
+ ]);
+ const text = await readTextAttachment(new File([vcf], "export.vcf"));
+ assert.match(text, /FN;CHARSET=windows-1252:Café/);
+ assert.match(text, /FN:Straße/);
+});
+
+test("a card whose value breaks its declaration still reads as one unit", async () => {
+ // Falling back has to keep working: the per-property read is all-or-nothing.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nNOTE:plain "),
+ 0xe9,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ // NOTE declares nothing and is not UTF-8, so the per-property read gives up
+ // and the single declaration is applied to the file as it was before.
+ assert.match(
+ await readTextAttachment(new File([vcf], "contact.vcf")),
+ /NOTE:plain é/,
+ );
+});
+
+test("a folded boundary parameter is still a boundary", async () => {
+ // "multipart/mixed;\r\n boundary=..." is an ordinary wrap. Missing it meant
+ // no part header was ever scanned, so the charset they declare went too.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode(
+ 'Content-Type: multipart/mixed;\r\n boundary="part"\r\n\r\n--part\r\n' +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...enc.encode("\r\n--part--\r\n"),
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "message.eml")), /Café/);
+});
+
+test("a nested boundary stops being one after its multipart closes", async () => {
+ // The inner delimiter is finished at "--inner--", so a sibling part repeating
+ // that line is body text and the header-shaped line below it declares nothing.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode(
+ "Content-Type: multipart/mixed; boundary=outer\r\n\r\n--outer\r\n" +
+ "Content-Type: multipart/alternative; boundary=inner\r\n\r\n--inner\r\n" +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ...enc.encode(
+ "\r\n--inner--\r\n--outer\r\n" +
+ "Content-Type: text/plain\r\n\r\n" +
+ "The sibling quotes the inner delimiter:\r\n--inner\r\n" +
+ "Content-Type: text/plain; charset=windows-1251\r\n--outer--\r\n",
+ ),
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "message.eml")), /Café/);
+});
+
+test("two spellings of one encoding are one declaration", async () => {
+ // windows-1252, CP1252 and latin1 are three names for the same decoder, so an
+ // archive using them in different messages is not a multi-charset one.
+ const enc = new TextEncoder();
+ for (const [first, second] of [
+ ["windows-1252", "CP1252"],
+ ["ISO-8859-1", "latin1"],
+ ] as const) {
+ const mbox = new Uint8Array([
+ ...enc.encode(
+ "From [email protected] Mon Jan 1 00:00:00 2024\r\n" +
+ `Content-Type: text/plain; charset=${first}\r\n\r\nCaf`,
+ ),
+ 0xe9,
+ ...enc.encode(
+ "\r\n\r\nFrom [email protected] Mon Jan 1 00:00:00 2024\r\n" +
+ `Content-Type: text/plain; charset=${second}\r\n\r\nna`,
+ ),
+ 0xef,
+ ...enc.encode("ve\r\n"),
+ ]);
+ const text = await readTextAttachment(new File([mbox], "inbox.mbox"));
+ assert.match(text, /Café/, `${first} + ${second}`);
+ assert.match(text, /naïve/, `${first} + ${second}`);
+ }
+ // Genuinely different encodings are still two.
+ const mixed = new Uint8Array([
+ ...enc.encode("Content-Type: text/plain; charset=windows-1252\r\n\r\nCaf"),
+ 0xe9,
+ ...enc.encode(
+ "\r\n\r\nFrom [email protected] Mon Jan 1 00:00:00 2024\r\n" +
+ "Content-Type: text/plain; charset=windows-1251\r\n\r\n",
+ ),
+ 0xcf,
+ 0xf0,
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([mixed], "inbox.mbox")),
+ (error: Error) => {
+ assert.match(error.message, /windows-1252, windows-1251/);
+ return true;
+ },
+ );
+});
+
+test("a wide XML declaration still names its encoding", async () => {
+ // The grammar puts no bound on the whitespace between the parts, so a fixed
+ // prefix could cut `encoding` off and refuse a file that states it plainly.
+ const enc = new TextEncoder();
+ const padding = " ".repeat(400);
+ const xml = new Uint8Array([
+ ...enc.encode(`<?xml version="1.0"${padding}encoding="windows-1252"?><t>Caf`),
+ 0xe9,
+ ...enc.encode("</t>"),
+ ]);
+ assert.match(await readTextAttachment(new File([xml], "ui.resx")), /Café/);
+ // A document that does not open with a declaration is untouched by the scan.
+ const plain = new Uint8Array([
+ ...enc.encode("<t>Caf"),
+ 0xc3,
+ 0xa9,
+ ...enc.encode("</t>"),
+ ]);
+ assert.match(await readTextAttachment(new File([plain], "ui.resx")), /Café/);
+});
+
+test("the audio reference picker reads a 3GP recording's tracks", async () => {
+ const { REFERENCE_PICKER_ACCEPT, referenceFileRejection } = await import(
+ "../src/features/video/reference-budget.ts"
+ );
+ // Offered by the dialog, then settled from the container once it is in hand.
+ assert.ok(REFERENCE_PICKER_ACCEPT.audio.split(",").includes(".3gp"));
+
+ const recording = new File([threeGpWithTracks(["soun"])], "voice.3gp", {
+ type: "",
+ });
+ const classifiedRecording = await classifiedAttachmentFile(recording);
+ assert.equal(referenceFileRejection("audio", classifiedRecording), null);
+ // And a real clip picked there is refused, rather than staged as audio.
+ const clip = new File([threeGpWithTracks(["soun", "vide"])], "clip.3gp", {
+ type: "",
+ });
+ assert.equal(
+ referenceFileRejection("audio", await classifiedAttachmentFile(clip)),
+ "Please choose an audio file",
+ );
+ // The video picker does not take the recording either, extension or not.
+ assert.equal(isVideoFile(classifiedRecording), false);
+ assert.equal(
+ referenceFileRejection("video", classifiedRecording),
+ "Please choose a video file",
+ );
+
+ const picker = readFileSync(
+ new URL("../src/features/video/reference-picker.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(picker, /await classifiedAttachmentFile\(picked\)/);
+});
+
+test("only a real charset parameter is a charset declaration", async () => {
+ // "name=" can carry a filename that reads like one. Taking the first match
+ // anywhere in the header stopped at the filename and never saw the parameter.
+ const enc = new TextEncoder();
+ const named = new Uint8Array([
+ ...enc.encode(
+ 'Content-Type: text/plain; name="charset=windows-1251.txt";' +
+ " charset=windows-1252\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ]);
+ assert.match(
+ await readTextAttachment(new File([named], "message.eml")),
+ /Café/,
+ );
+ // A filename naming a supported label would otherwise pick the wrong decoder
+ // silently, which is the worse half of the same bug.
+ const plausible = new Uint8Array([
+ ...enc.encode(
+ 'Content-Type: text/plain; name="charset=windows-1251";' +
+ " charset=windows-1252\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ]);
+ assert.match(
+ await readTextAttachment(new File([plausible], "message.eml")),
+ /Café/,
+ );
+ // An ordinary header is unaffected, quoted or not.
+ for (const header of [
+ "Content-Type: text/plain; charset=windows-1252",
+ 'Content-Type: text/plain; charset="windows-1252"',
+ "Content-Type: text/plain;charset=windows-1252",
+ "Content-Type: text/plain; CHARSET = windows-1252",
+ ]) {
+ const eml = new Uint8Array([
+ ...enc.encode(`${header}\r\n\r\nCaf`),
+ 0xe9,
+ ]);
+ assert.match(
+ await readTextAttachment(new File([eml], "message.eml")),
+ /Café/,
+ header,
+ );
+ }
+});
+
+test("a truncated vCard preview reads per property too", async () => {
+ // The preview pane decodes a prefix. Applying the sole declaration to all of
+ // it showed Café for text the sent attachment renders as Café.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nEND:VCARD\r\nBEGIN:VCARD\r\nVERSION:3.0\r\nFN:Stra"),
+ 0xc3,
+ 0x9f,
+ ...enc.encode("e\r\nEND:VCARD\r\n"),
+ ]);
+ const preview = decodeTextAttachmentBytes(vcf, "export.vcf", true);
+ assert.match(preview, /FN;CHARSET=windows-1252:Café/);
+ assert.match(preview, /FN:Straße/);
+ assert.equal(preview, await readTextAttachment(new File([vcf], "export.vcf")));
+
+ // A prefix cut through a character drops it rather than failing the read.
+ const cut = vcf.subarray(0, vcf.length - 12);
+ const cutPreview = decodeTextAttachmentBytes(cut, "export.vcf", true);
+ assert.match(cutPreview, /FN;CHARSET=windows-1252:Café/);
+});
+
+test("a 3GP is inspected however large the surface taking it allows", async () => {
+ // The ceiling here was the composer's video cap, and the video reference
+ // surface accepts a larger file, so a recording in between was staged as a
+ // video reference on its extension alone.
+ const { MAX_REFERENCE_BYTES: limits } = await import(
+ "../src/features/video/reference-budget.ts"
+ );
+ const recording = threeGpWithTracks(["soun"]);
+ const oversize = new File([recording], "voice.3gp", { type: "" });
+ Object.defineProperty(oversize, "size", {
+ value: Math.max(limits.video, limits.audio, MAX_TEXT_ATTACHMENT_BYTES) + 1,
+ });
+ const classified = await classifiedAttachmentFile(oversize);
+ assert.equal(classified.type, "audio/3gpp");
+ assert.equal(isVideoFile(classified), false);
+
+ // Nothing in the predicate turns on size any more, so no surface's limit can
+ // drift past it again.
+ const source = readFileSync(
+ new URL("../src/lib/video-utils.ts", import.meta.url),
+ "utf8",
+ );
+ assert.match(
+ source,
+ /export function needsAttachmentTrackInspection[^)]*\)[^{]*\{\s*return \/\\\.3gp\$\/i\.test\(file\.name\);\s*\}/,
+ );
+});
+
+test("a boundary counts only on the header that owns the parts", async () => {
+ // A quoted filename on any header could register a delimiter, and a body line
+ // repeating it then reopened the headers.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode(
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n" +
+ 'Content-Disposition: attachment; filename="report; boundary=fake"\r\n\r\nCaf',
+ ),
+ 0xe9,
+ ...enc.encode(
+ "\r\n--fake\r\nContent-Type: text/plain; charset=windows-1251\r\n",
+ ),
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "message.eml")), /Café/);
+
+ // A real multipart boundary still registers, quoted or bare.
+ for (const header of [
+ 'Content-Type: multipart/mixed; boundary="part"',
+ "Content-Type: multipart/alternative; boundary=part",
+ ]) {
+ const multipart = new Uint8Array([
+ ...enc.encode(
+ `${header}\r\n\r\n--part\r\nContent-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf`,
+ ),
+ 0xe9,
+ ]);
+ assert.match(
+ await readTextAttachment(new File([multipart], "message.eml")),
+ /Café/,
+ header,
+ );
+ }
+});
+
+test("a preview finds a declaration that sits past its own slice", async () => {
+ // The preview decodes a bounded prefix. Looking for the declaration inside
+ // that prefix reported an error for a file the attachment itself decodes.
+ const enc = new TextEncoder();
+ // A translator comment in the file's own encoding, so the prefix is not UTF-8
+ // and the strict decode has to fall back to the declaration to read it.
+ const comment = new Uint8Array([
+ ...enc.encode("# "),
+ 0xcf,
+ 0xf0,
+ ...enc.encode(" translated by\n"),
+ ]);
+ const whole = new Uint8Array([
+ ...Array.from({ length: 200 }).flatMap(() => Array.from(comment)),
+ ...enc.encode(
+ `msgid ""\nmsgstr ""\n` +
+ '"Content-Type: text/plain; charset=windows-1251\\n"\n\n' +
+ 'msgid "hi"\nmsgstr "',
+ ),
+ 0xcf,
+ 0xf0,
+ ...enc.encode('"\n'),
+ ]);
+ // The slice stops above the header entry, exactly as the preview cap would.
+ const slice = whole.subarray(0, comment.length * 200 - 40);
+ assert.throws(() => decodeTextAttachmentBytes(slice, "ru.po", true));
+ assert.equal(
+ typeof decodeTextAttachmentBytes(slice, "ru.po", true, whole),
+ "string",
+ );
+
+ const source = readFileSync(
+ new URL("../src/features/chat/attachment-content.ts", import.meta.url),
+ "utf8",
+ );
+ assert.match(source, /decodeTextAttachmentBytes\(bytes, file\.name, truncated, whole\)/);
+ assert.match(source, /DECLARES_ITS_CHARSET_RE/);
+});
+
+test("a 3GP typed as audio by the platform is still inspected", async () => {
+ // A platform that maps the shared extension to audio/3gpp says so for a clip
+ // too, and the audio adapter is matched before the video one.
+ const clip = new File([threeGpWithTracks(["soun", "vide"])], "clip.3gp", {
+ type: "audio/3gpp",
+ });
+ const classified = await classifiedAttachmentFile(clip);
+ assert.equal(classified.type, "video/3gpp");
+ assert.equal(isVideoFile(classified), true);
+ assert.equal(isAudioAttachmentFile(classified), false);
+
+ // A recording already typed correctly is returned untouched, not rewrapped.
+ const recording = new File([threeGpWithTracks(["soun"])], "voice.3gp", {
+ type: "audio/3gpp",
+ });
+ assert.equal(await classifiedAttachmentFile(recording), recording);
+
+ // Tracks that cannot be read decide nothing.
+ const unreadable = new File([new Uint8Array([1, 2, 3, 4])], "odd.3gp", {
+ type: "audio/3gpp",
+ });
+ assert.equal(await classifiedAttachmentFile(unreadable), unreadable);
+});
+
+test("the reference drop zone takes what its dialog offers", async () => {
+ const { REFERENCE_DROP_ACCEPT, REFERENCE_PICKER_ACCEPT: picker } = await import(
+ "../src/features/video/reference-budget.ts"
+ );
+ // The zone filters on the name before the classifier can look at the file, so
+ // a list narrower than the dialog's refused what the button accepts.
+ assert.ok(REFERENCE_DROP_ACCEPT.audio.split(",").includes(".3gp"));
+ for (const [kind, offered] of Object.entries(picker)) {
+ const dropped = REFERENCE_DROP_ACCEPT[kind as "audio" | "video"].split(",");
+ for (const entry of offered.split(",")) {
+ if (!entry.startsWith(".")) continue;
+ assert.ok(dropped.includes(entry), `${kind} ${entry}`);
+ }
+ // Extensions only: the zone shows this list verbatim when it refuses a file.
+ assert.equal(dropped.some((entry) => entry.includes("/")), false, kind);
+ }
+ // Nothing the chat drop lists may be missing from it either.
+ for (const ext of CHAT_AUDIO_DROP_ACCEPT.split(",")) {
+ assert.ok(REFERENCE_DROP_ACCEPT.audio.split(",").includes(ext), ext);
+ }
+ for (const ext of CHAT_VIDEO_DROP_ACCEPT.split(",")) {
+ assert.ok(REFERENCE_DROP_ACCEPT.video.split(",").includes(ext), ext);
+ }
+
+ const picked = readFileSync(
+ new URL("../src/features/video/reference-picker.tsx", import.meta.url),
+ "utf8",
+ );
+ assert.match(picked, /accept: REFERENCE_DROP_ACCEPT\[kind\]/);
+});
+
+test("a quoted parameter value resolves its escapes", async () => {
+ // The splitter honours "\\X" already; the unquoting did not, so a value
+ // carrying a quote stopped at the escaped one and kept the backslashes.
+ const enc = new TextEncoder();
+ const eml = new Uint8Array([
+ ...enc.encode(
+ 'Content-Type: multipart/mixed; boundary="part\\"one"\r\n\r\n--part"one\r\n' +
+ "Content-Type: text/plain; charset=ISO-8859-1\r\n\r\nCaf",
+ ),
+ 0xe9,
+ ]);
+ assert.match(await readTextAttachment(new File([eml], "message.eml")), /Café/);
+
+ // And a parameter after the escaped quote is still found, rather than being
+ // swallowed by a value that never terminated.
+ const named = new Uint8Array([
+ ...enc.encode(
+ 'Content-Type: text/plain; name="say \\"hi\\""; charset=windows-1252\r\n\r\nCaf',
+ ),
+ 0xe9,
+ ]);
+ assert.match(await readTextAttachment(new File([named], "message.eml")), /Café/);
+});
+
+test("the clipboard reader takes what the native side hands it", () => {
+ // Rust reads a pasted clip to MAX_CLIPBOARD_VIDEO_BYTES; refusing it here
+ // threw away a file already read and encoded, and dropped the paste with it.
+ const source = readFileSync(
+ new URL("../src/features/chat/utils/clipboard-files.ts", import.meta.url),
+ "utf8",
+ );
+ const rust = readFileSync(
+ new URL("../../src-tauri/src/native_clipboard.rs", import.meta.url),
+ "utf8",
+ );
+ const rustLimit = (name: string): number => {
+ const raw = rust.match(
+ new RegExp(`const ${name}: u64 = ([0-9_]+(?:\\\\s*\\\\*\\\\s*[0-9_]+)*)`),
+ )?.[1];
+ assert.ok(raw, name);
+ return raw
+ .split("*")
+ .reduce((total, part) => total * Number(part.replace(/[\s_]/g, "")), 1);
+ };
+ const frontLimit = (name: string): number => {
+ const raw = source.match(
+ new RegExp(`const ${name} = ([0-9_]+(?:\\\\s*\\\\*\\\\s*[0-9_]+)*)`),
+ )?.[1];
+ assert.ok(raw, name);
+ return raw
+ .split("*")
+ .reduce((total, part) => total * Number(part.replace(/[\s_]/g, "")), 1);
+ };
+
+ assert.equal(
+ frontLimit("MAX_CLIPBOARD_VIDEO_BYTES"),
+ rustLimit("MAX_CLIPBOARD_VIDEO_BYTES"),
+ );
+ assert.equal(
+ frontLimit("MAX_CLIPBOARD_NON_AUDIO_BYTES"),
+ rustLimit("MAX_CLIPBOARD_SOURCE_BYTES"),
+ );
+ // The total may not refuse a single file the per-file limits allow.
+ assert.match(source, /const MAX_CLIPBOARD_BYTES = MAX_CLIPBOARD_VIDEO_BYTES;/);
+ assert.match(rust, /const MAX_CLIPBOARD_TOTAL_BYTES: u64 = MAX_CLIPBOARD_VIDEO_BYTES;/);
+ // And video is classified before the catch-all, as it is natively.
+ assert.match(
+ source,
+ /file\.mimeType\.startsWith\("video\/"\)\s*\?\s*MAX_CLIPBOARD_VIDEO_BYTES/,
+ );
+});
+
+test("an unsupported charset is refused however many others sit beside it", async () => {
+ // The per-property read cannot honour UTF-7, and the fallback then read the
+ // card as UTF-8 and accepted it with the escapes literal, because every other
+ // byte in it happened to be ASCII. The same declaration alone was refused, so
+ // one card's fate turned on how many of its neighbours declared anything.
+ const card = (extra: string) =>
+ new File(
+ [
+ `BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=UTF-7:Bj+APg-rk\r\n${extra}END:VCARD\r\n`,
+ ],
+ "contact.vcf",
+ );
+ const message = /declares charset "UTF-7", which this browser has no decoder for/;
+ await assert.rejects(readTextAttachment(card("")), (error: Error) => {
+ assert.match(error.message, message);
+ return true;
+ });
+ await assert.rejects(
+ readTextAttachment(card("NOTE;CHARSET=windows-1252:plain ascii\r\n")),
+ (error: Error) => {
+ assert.match(error.message, message);
+ return true;
+ },
+ );
+});
+
+test("a property declaring two charsets is refused rather than read as UTF-8", async () => {
+ const vcf = new File(
+ [
+ "BEGIN:VCARD\r\nVERSION:2.1\r\n" +
+ "FN;CHARSET=windows-1252;CHARSET=windows-1251:ascii\r\n" +
+ "NOTE;CHARSET=utf-8:ascii\r\nEND:VCARD\r\n",
+ ],
+ "contact.vcf",
+ );
+ await assert.rejects(readTextAttachment(vcf), (error: Error) => {
+ assert.match(
+ error.message,
+ /One of its properties declares two charsets \(windows-1252, windows-1251\)/,
+ );
+ return true;
+ });
+});
+
+test("a card whose undeclared property is legacy still reports its charsets", async () => {
+ // Nothing about the declarations blocks this reading: it is an undeclared
+ // property holding bytes that are not UTF-8. That case keeps the answer it
+ // had, so the change only reaches files a declaration itself defeats.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode(
+ "BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:ok\r\nORG;CHARSET=windows-1251:ok\r\nNOTE:Caf",
+ ),
+ 0xe9,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ await assert.rejects(
+ readTextAttachment(new File([vcf], "contact.vcf")),
+ (error: Error) => {
+ assert.match(error.message, /windows-1252, windows-1251/);
+ return true;
+ },
+ );
+});
+
+test("a card every property of which reads still decodes per property", async () => {
+ // The guard above must not refuse cards the per-property read handles.
+ const enc = new TextEncoder();
+ const vcf = new Uint8Array([
+ ...enc.encode("BEGIN:VCARD\r\nVERSION:2.1\r\nFN;CHARSET=windows-1252:Caf"),
+ 0xe9,
+ ...enc.encode("\r\nORG;CHARSET=windows-1251:"),
+ 0xcf,
+ 0xf0,
+ ...enc.encode("\r\nEND:VCARD\r\n"),
+ ]);
+ const text = await readTextAttachment(new File([vcf], "contact.vcf"));
+ assert.match(text, /FN;CHARSET=windows-1252:Café/);
+ assert.match(text, /ORG;CHARSET=windows-1251:Пр/);
+});
+
+test("a processing instruction is not read as the document's declaration", async () => {
+ // `xml-stylesheet` and `xml-model` open with the same five bytes as a
+ // declaration. Taking one for the prolog decoded a UTF-8 document as a code
+ // page and returned mojibake, which no error accompanies.
+ const doc =
+ '<?xml-stylesheet type="text/xsl" href="s.xsl" encoding="windows-1252"?>\n' +
+ "<note>café naïve</note>\n";
+ const text = await readTextAttachment(new File([doc], "doc.xml"));
+ assert.match(text, /café naïve/);
+});
+
+test("a declaration still needs only the whitespace its grammar requires", async () => {
+ // The check is a sixth byte of XML's own S, so tab and newline pass with space.
+ const enc = new TextEncoder();
+ for (const space of [" ", "\t", "\n", "\r"]) {
+ const xml = new Uint8Array([
+ ...enc.encode(`<?xml${space}version="1.0" encoding="windows-1252"?><t>Caf`),
+ 0xe9,
+ ...enc.encode("</t>"),
+ ]);
+ assert.match(
+ await readTextAttachment(new File([xml], "ui.resx")),
+ /Café/,
+ `whitespace ${JSON.stringify(space)}`,
+ );
+ }
+});
+
+test("a gettext charset split across two literals reads as one label", async () => {
+ // PO concatenates adjacent quoted pieces, so the header is not the source
+ // text. Matching the raw entry read `windows-` and refused the catalog for a
+ // charset it does not declare.
+ const enc = new TextEncoder();
+ const po = new Uint8Array([
+ ...enc.encode(
+ 'msgid ""\nmsgstr ""\n' +
+ '"Project-Id-Version: demo\\n"\n' +
+ '"Content-Type: text/plain; charset=windows-"\n' +
+ '"1252\\n"\n\n' +
+ 'msgid "cafe"\nmsgstr "caf',
+ ),
+ 0xe9,
+ ...enc.encode('"\n'),
+ ]);
+ assert.match(await readTextAttachment(new File([po], "messages.po")), /café/);
+});
+
+test("a header entry the prefix cuts is answered from the whole file", async () => {
+ // The same truncation by a different route: the 64 KiB prefix ends inside the
+ // charset, and the last complete piece stops at `windows-`. An entry that runs
+ // to the cut has to be re-read rather than answered from half a value.
+ const enc = new TextEncoder();
+ const head = 'msgid ""\nmsgstr ""\n"Content-Type: text/plain; charset=windows-';
+ const comment = "# " + "c".repeat(64 * 1024 - head.length - 4) + "\n";
+ const source = comment + head;
+ // The cut lands inside the value, with the rest of it in the next piece.
+ assert.ok(source.length > 64 * 1024 - 8 && source.length <= 64 * 1024);
+ const po = new Uint8Array([
+ ...enc.encode(`${source}"\n"1252\\n"\n\nmsgid "c"\nmsgstr "caf`),
+ 0xe9,
+ ...enc.encode('"\n'),
+ ]);
+ assert.match(await readTextAttachment(new File([po], "big.po")), /café/);
+});
+
+test("escapes in a header entry resolve before the charset is read", async () => {
+ // `\n` is a newline, not the letter: emitting it literally glued the charset
+ // to the header that follows and made `utf-8` read as `utf-8nContent`.
+ const po =
+ 'msgid ""\nmsgstr ""\n' +
+ '"Content-Type: text/plain; charset=utf-8\\nContent-Transfer-Encoding: 8bit\\n"\n\n' +
+ 'msgid "c"\nmsgstr "café"\n';
+ assert.match(await readTextAttachment(new File([po], "m.po")), /café/);
+});
diff --git a/studio/frontend/tests/pr9057-video-simulation.test.ts b/studio/frontend/tests/pr9057-video-simulation.test.ts
index 9c80916b224..3f9730ccd29 100644
--- a/studio/frontend/tests/pr9057-video-simulation.test.ts
+++ b/studio/frontend/tests/pr9057-video-simulation.test.ts
@@ -189,6 +189,11 @@ test("an m4a keeps going to audio even though mp4 is a video container", () => {
assert.equal(dispatch({ name: "voice.m4a", type: "audio/mp4" }), "audio");
});
+test("an audio-only 3gp reaches audio while a video 3gp stays video", () => {
+ assert.equal(dispatch({ name: "voice.3gp", type: "audio/3gpp" }), "audio");
+ assert.equal(dispatch({ name: "clip.3gp", type: "video/3gpp" }), "video");
+});
+
test("adding the video adapter did not steal any pre-existing attachment type", () => {
// Same corpus, with the video adapter removed: the answer must be unchanged
// for everything that is not a video.
diff --git a/studio/src-tauri/src/native_clipboard.rs b/studio/src-tauri/src/native_clipboard.rs
index 810773ee1ee..b2e2d588edd 100644
--- a/studio/src-tauri/src/native_clipboard.rs
+++ b/studio/src-tauri/src/native_clipboard.rs
@@ -1,6 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+use crate::native_path_policy::{
+ is_audio_only_3gp, is_binary_property_list, is_binary_tracker_mod, is_binary_vobsub,
+ is_binary_office_template, is_compiled_fortran_mod,
+};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use serde::Serialize;
use std::fs::File;
@@ -12,7 +16,10 @@ const MAX_CLIPBOARD_RGBA_BYTES: u64 = 64 * 1024 * 1024;
const MAX_CLIPBOARD_PNG_BYTES: usize = 20 * 1024 * 1024;
const MAX_CLIPBOARD_SOURCE_BYTES: u64 = 20 * 1024 * 1024;
const MAX_CLIPBOARD_AUDIO_BYTES: u64 = 25 * 1024 * 1024;
-const MAX_CLIPBOARD_TOTAL_BYTES: u64 = MAX_CLIPBOARD_AUDIO_BYTES;
+// Mirrors MAX_NATIVE_VIDEO_BYTES in native_intents.rs: a pasted clip and a
+// dropped one are the same file, and the 20 MB source cap skipped most videos.
+const MAX_CLIPBOARD_VIDEO_BYTES: u64 = 75_497_280;
+const MAX_CLIPBOARD_TOTAL_BYTES: u64 = MAX_CLIPBOARD_VIDEO_BYTES;
const MAX_CLIPBOARD_FILES: usize = 8;
const MAX_CLIPBOARD_CANDIDATES: usize = 32;
#[cfg(target_os = "linux")]
@@ -62,6 +69,9 @@ fn validate_png_bytes(png: &[u8]) -> Result<(), String> {
}
fn clipboard_file_mime_type(path: &Path) -> Option<&'static str> {
+ if crate::native_path_policy::is_text_attachment_name(path) {
+ return Some("text/plain");
+ }
let extension = path
.extension()
.and_then(|value| value.to_str())
@@ -72,30 +82,46 @@ fn clipboard_file_mime_type(path: &Path) -> Option<&'static str> {
"md" | "markdown" | "mdx" => "text/markdown",
"csv" => "text/csv",
"html" | "htm" => "text/html",
- "xml" => "application/xml",
+ "xml" | "plist" => "application/xml",
"svg" => "image/svg+xml",
"jpg" | "jpeg" => "image/jpeg",
"png" => "image/png",
"webp" => "image/webp",
"gif" => "image/gif",
+ "mp4" => "video/mp4",
+ "m4v" => "video/x-m4v",
+ "mov" => "video/quicktime",
+ "webm" => "video/webm",
+ "mkv" => "video/x-matroska",
+ "avi" => "video/x-msvideo",
+ "mpg" | "mpeg" => "video/mpeg",
+ "wmv" => "video/x-ms-wmv",
+ "flv" => "video/x-flv",
+ "ogv" => "video/ogg",
"pdf" => "application/pdf",
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"odt" => "application/vnd.oasis.opendocument.text",
"ods" => "application/vnd.oasis.opendocument.spreadsheet",
- "mp3" => "audio/mpeg",
+ "mp3" | "mp2" => "audio/mpeg",
"wav" => "audio/wav",
"m4a" => "audio/mp4",
"ogg" | "oga" => "audio/ogg",
+ "opus" => "audio/opus",
"flac" => "audio/flac",
"aac" => "audio/aac",
- "txt" | "text" | "log" | "rst" | "tsv" | "yaml" | "yml" | "toml" | "ini" | "cfg"
- | "conf" | "env" | "properties" | "css" | "scss" | "sass" | "less" | "js" | "jsx"
- | "mjs" | "cjs" | "ts" | "tsx" | "py" | "pyi" | "ipynb" | "rb" | "php" | "go" | "rs"
- | "java" | "kt" | "kts" | "scala" | "swift" | "c" | "h" | "cc" | "cpp" | "hpp" | "cxx"
- | "cs" | "m" | "mm" | "sh" | "bash" | "zsh" | "fish" | "ps1" | "bat" | "lua" | "pl"
- | "pm" | "r" | "jl" | "dart" | "vue" | "svelte" | "astro" | "sql" | "graphql" | "gql"
- | "proto" | "tf" | "tfvars" | "gradle" | "dockerfile" | "makefile" | "cmake" | "diff"
- | "patch" => "text/plain",
+ "aiff" | "aif" | "aifc" => "audio/aiff",
+ "caf" => "audio/x-caf",
+ "wma" => "audio/x-ms-wma",
+ "amr" => "audio/amr",
+ // Provisional until the BMFF handlers can be inspected after reading.
+ "3gp" => "video/3gpp",
+ "vtt" => "text/vtt",
+ "srt" => "application/x-subrip",
+ // .txt is a RAG type, so it is absent from TEXT_ATTACHMENT_EXTS and
+ // named here; the rest of the source and text formats come from the one
+ // list the composer and the drop path already share.
+ "txt" => "text/plain",
+ other if crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&other) => "text/plain",
_ => return None,
};
Some(mime_type)
@@ -104,6 +130,8 @@ fn clipboard_file_mime_type(path: &Path) -> Option<&'static str> {
fn clipboard_file_max_bytes(mime_type: &str) -> u64 {
if mime_type.starts_with("audio/") {
MAX_CLIPBOARD_AUDIO_BYTES
+ } else if mime_type.starts_with("video/") {
+ MAX_CLIPBOARD_VIDEO_BYTES
} else {
MAX_CLIPBOARD_SOURCE_BYTES
}
@@ -152,7 +180,20 @@ fn read_clipboard_files(paths: Vec<PathBuf>) -> Result<Vec<NativeClipboardFile>,
let Ok(metadata) = source.metadata() else {
continue;
};
- let limit = clipboard_file_max_bytes(mime_type).min(remaining);
+ let is_3gp = path
+ .extension()
+ .and_then(|value| value.to_str())
+ .is_some_and(|extension| extension.eq_ignore_ascii_case("3gp"));
+ // A 3GP recording cannot use its final size limit until its BMFF
+ // handlers have been read and classified as audio-only or video. Read it
+ // under the larger of the two, as the drop path does; the audio cap is
+ // reapplied below once the track handlers say it is audio-only.
+ let provisional_limit = if is_3gp {
+ MAX_CLIPBOARD_VIDEO_BYTES
+ } else {
+ clipboard_file_max_bytes(mime_type)
+ };
+ let limit = provisional_limit.min(remaining);
if !metadata.is_file() || metadata.len() == 0 || metadata.len() > limit {
continue;
}
@@ -163,6 +204,22 @@ fn read_clipboard_files(paths: Vec<PathBuf>) -> Result<Vec<NativeClipboardFile>,
{
continue;
}
+ if is_binary_property_list(&path, &bytes)
+ || is_binary_vobsub(&path, &bytes)
+ || is_binary_tracker_mod(&path, &bytes)
+ || is_compiled_fortran_mod(&path, &bytes)
+ || is_binary_office_template(&path, &bytes)
+ {
+ continue;
+ }
+ let mime_type = if is_3gp && is_audio_only_3gp(&bytes) {
+ "audio/3gpp"
+ } else {
+ mime_type
+ };
+ if bytes.len() as u64 > clipboard_file_max_bytes(mime_type).min(remaining) {
+ continue;
+ }
remaining -= bytes.len() as u64;
files.push(NativeClipboardFile {
name,
@@ -362,6 +419,21 @@ pub async fn read_native_clipboard_png(
mod tests {
use super::*;
+ fn bmff_box(kind: &[u8; 4], payload: &[u8]) -> Vec<u8> {
+ let mut boxed = Vec::with_capacity(payload.len() + 8);
+ boxed.extend_from_slice(&((payload.len() + 8) as u32).to_be_bytes());
+ boxed.extend_from_slice(kind);
+ boxed.extend_from_slice(payload);
+ boxed
+ }
+
+ fn three_gp_with_handler(handler: &[u8; 4]) -> Vec<u8> {
+ let mut hdlr_payload = vec![0; 8];
+ hdlr_payload.extend_from_slice(handler);
+ let mdia = bmff_box(b"mdia", &bmff_box(b"hdlr", &hdlr_payload));
+ bmff_box(b"moov", &bmff_box(b"trak", &mdia))
+ }
+
#[test]
fn clipboard_dimensions_are_bounded() {
assert!(validate_dimensions(3840, 2160).is_ok());
@@ -380,9 +452,37 @@ mod tests {
clipboard_file_mime_type(Path::new("notes.md")),
Some("text/markdown")
);
+ assert_eq!(
+ clipboard_file_mime_type(Path::new("Containerfile")),
+ Some("text/plain")
+ );
+ assert_eq!(
+ clipboard_file_mime_type(Path::new("recording.3gp")),
+ Some("video/3gpp")
+ );
assert_eq!(clipboard_file_mime_type(Path::new("unknown.bin")), None);
}
+ #[test]
+ fn clipboard_file_mime_types_cover_chat_video_attachments() {
+ for (name, mime_type) in [
+ ("clip.mp4", "video/mp4"),
+ ("clip.m4v", "video/x-m4v"),
+ ("clip.mov", "video/quicktime"),
+ ("clip.webm", "video/webm"),
+ ("clip.mkv", "video/x-matroska"),
+ ("clip.avi", "video/x-msvideo"),
+ ("clip.mpg", "video/mpeg"),
+ ("clip.mpeg", "video/mpeg"),
+ ("clip.wmv", "video/x-ms-wmv"),
+ ("clip.flv", "video/x-flv"),
+ ("clip.ogv", "video/ogg"),
+ ("clip.3gp", "video/3gpp"),
+ ] {
+ assert_eq!(clipboard_file_mime_type(Path::new(name)), Some(mime_type));
+ }
+ }
+
#[cfg(target_os = "linux")]
#[test]
fn clipboard_png_headers_are_bounded_before_decode() {
@@ -420,9 +520,104 @@ mod tests {
assert_eq!(BASE64.decode(&files[0].base64).unwrap(), b"clipboard text");
}
+ #[test]
+ fn clipboard_skips_binary_property_lists_but_keeps_text_forms() {
+ let directory = tempfile::tempdir().unwrap();
+ let binary = directory.path().join("binary.plist");
+ std::fs::write(&binary, b"bplist00payload").unwrap();
+ let binary_strings = directory.path().join("binary.strings");
+ std::fs::write(&binary_strings, b"bplist00payload").unwrap();
+ let xml = directory.path().join("xml.plist");
+ std::fs::write(&xml, b"<?xml version=\"1.0\"?><plist/>").unwrap();
+ let text_strings = directory.path().join("Localizable.strings");
+ std::fs::write(&text_strings, b"\"hello\" = \"world\";").unwrap();
+
+ let files = read_clipboard_files(vec![binary, binary_strings, xml, text_strings]).unwrap();
+ assert_eq!(files.len(), 2);
+ assert_eq!(files[0].name, "xml.plist");
+ assert_eq!(files[0].mime_type, "application/xml");
+ assert_eq!(files[1].name, "Localizable.strings");
+ assert_eq!(files[1].mime_type, "text/plain");
+ }
+
+ #[test]
+ fn clipboard_skips_binary_vobsub_but_keeps_text_subtitles() {
+ let directory = tempfile::tempdir().unwrap();
+ let binary = directory.path().join("binary.sub");
+ std::fs::write(&binary, b"\x00\x00\x01\xbapayload").unwrap();
+ let text = directory.path().join("text.sub");
+ std::fs::write(&text, b"{1}{24}Subtitle").unwrap();
+
+ let files = read_clipboard_files(vec![binary, text]).unwrap();
+ assert_eq!(files.len(), 1);
+ assert_eq!(files[0].name, "text.sub");
+ assert_eq!(files[0].mime_type, "text/plain");
+ }
+
+ #[test]
+ fn clipboard_skips_tracker_mod_but_keeps_text_module_files() {
+ let directory = tempfile::tempdir().unwrap();
+ let binary = directory.path().join("track.mod");
+ let mut tracker = vec![0u8; 1084];
+ tracker[1080..].copy_from_slice(b"M.K.");
+ std::fs::write(&binary, tracker).unwrap();
+ let markerless = directory.path().join("classic.mod");
+ let mut soundtracker = vec![0u8; 600 + 1024 + 8];
+ soundtracker[43] = 4;
+ soundtracker[45] = 64;
+ soundtracker[470] = 1;
+ soundtracker[471] = 120;
+ std::fs::write(&markerless, soundtracker).unwrap();
+ let text = directory.path().join("go.mod");
+ std::fs::write(&text, b"module example.com/project\n").unwrap();
+
+ let files = read_clipboard_files(vec![binary, markerless, text]).unwrap();
+ assert_eq!(files.len(), 1);
+ assert_eq!(files[0].name, "go.mod");
+ assert_eq!(files[0].mime_type, "text/plain");
+ }
+
+ #[test]
+ fn clipboard_3gp_uses_its_bmff_track_type() {
+ let directory = tempfile::tempdir().unwrap();
+ let audio = directory.path().join("recording.3gp");
+ std::fs::write(&audio, three_gp_with_handler(b"soun")).unwrap();
+ let video = directory.path().join("video.3gp");
+ std::fs::write(&video, three_gp_with_handler(b"vide")).unwrap();
+
+ let files = read_clipboard_files(vec![audio, video]).unwrap();
+ assert_eq!(files.len(), 2);
+ assert_eq!(files[0].mime_type, "audio/3gpp");
+ assert_eq!(files[1].mime_type, "video/3gpp");
+ }
+
#[test]
fn clipboard_audio_uses_the_chat_upload_boundary() {
- assert_eq!(clipboard_file_max_bytes("audio/mpeg"), 25 * 1024 * 1024);
+ assert_eq!(
+ clipboard_file_max_bytes("audio/mpeg"),
+ MAX_CLIPBOARD_AUDIO_BYTES
+ );
+ // A video gets the video budget, not the 20 MB source cap, which skipped
+ // most clips the picker and the drop path both accept.
+ assert_eq!(
+ clipboard_file_max_bytes("video/3gpp"),
+ MAX_CLIPBOARD_VIDEO_BYTES
+ );
+ assert_eq!(
+ clipboard_file_max_bytes("video/mp4"),
+ MAX_CLIPBOARD_VIDEO_BYTES
+ );
+ assert!(MAX_CLIPBOARD_TOTAL_BYTES >= MAX_CLIPBOARD_VIDEO_BYTES);
+ // The pasted and dropped limits are the same file's limit.
+ let intents = include_str!("native_intents.rs");
+ let native = intents
+ .split("const MAX_NATIVE_VIDEO_BYTES: u64 = ")
+ .nth(1)
+ .and_then(|rest| rest.split(';').next())
+ .map(|value| value.replace('_', ""))
+ .and_then(|value| value.parse::<u64>().ok())
+ .expect("native video cap not found");
+ assert_eq!(native, MAX_CLIPBOARD_VIDEO_BYTES);
assert_eq!(
clipboard_file_max_bytes("text/markdown"),
MAX_CLIPBOARD_SOURCE_BYTES
diff --git a/studio/src-tauri/src/native_intents.rs b/studio/src-tauri/src/native_intents.rs
index d39a879a784..f6faeac9989 100644
--- a/studio/src-tauri/src/native_intents.rs
+++ b/studio/src-tauri/src/native_intents.rs
@@ -5,8 +5,9 @@ use crate::native_backend_lease::{
};
use crate::native_path_policy::{
classify_artifact_path, classify_native_attachment_path, classify_native_dataset_path,
- classify_native_document_folder, classify_native_model_path, reveal_target, ClassifiedPath,
- NativeArtifactKind,
+ classify_native_document_folder, classify_native_model_path, is_audio_only_3gp,
+ is_binary_property_list, is_binary_tracker_mod, is_binary_vobsub, is_binary_office_template, is_compiled_fortran_mod, is_text_attachment_name,
+ reveal_target, ClassifiedPath, NativeArtifactKind,
};
use serde::Serialize;
use std::collections::{HashMap, VecDeque};
@@ -626,6 +627,9 @@ pub struct NativeAttachmentFile {
}
fn attachment_mime_type(path: &Path) -> Option<&'static str> {
+ if is_text_attachment_name(path) {
+ return Some("text/plain");
+ }
let ext = path.extension()?.to_str()?.to_ascii_lowercase();
match ext.as_str() {
"jpg" | "jpeg" => Some("image/jpeg"),
@@ -633,22 +637,42 @@ fn attachment_mime_type(path: &Path) -> Option<&'static str> {
"webp" => Some("image/webp"),
"gif" => Some("image/gif"),
"wav" => Some("audio/wav"),
- "mp3" => Some("audio/mpeg"),
+ "mp3" | "mp2" => Some("audio/mpeg"),
"m4a" => Some("audio/mp4"),
"ogg" | "oga" => Some("audio/ogg"),
+ "opus" => Some("audio/opus"),
"flac" => Some("audio/flac"),
+ "aac" => Some("audio/aac"),
+ "aiff" | "aif" | "aifc" => Some("audio/aiff"),
+ "caf" => Some("audio/x-caf"),
+ "wma" => Some("audio/x-ms-wma"),
+ "amr" => Some("audio/amr"),
"mp4" => Some("video/mp4"),
+ "m4v" => Some("video/x-m4v"),
"mov" => Some("video/quicktime"),
"webm" => Some("video/webm"),
"mkv" => Some("video/x-matroska"),
"avi" => Some("video/x-msvideo"),
+ "mpg" | "mpeg" => Some("video/mpeg"),
+ "wmv" => Some("video/x-ms-wmv"),
+ "flv" => Some("video/x-flv"),
+ "3gp" => Some("video/3gpp"),
+ "ogv" => Some("video/ogg"),
"ods" => Some("application/vnd.oasis.opendocument.spreadsheet"),
"odt" => Some("application/vnd.oasis.opendocument.text"),
// Stamped like native_clipboard.rs.
- "json" | "jsonl" | "ndjson" => Some("application/json"),
- "mdx" => Some("text/markdown"),
+ "json" | "jsonl" | "ndjson" | "jsonc" | "json5" | "geojson" | "har" | "avsc"
+ | "tfstate" => Some("application/json"),
+ "mdx" | "rmd" | "qmd" => Some("text/markdown"),
"csv" => Some("text/csv"),
- "xml" => Some("application/xml"),
+ "tsv" => Some("text/tab-separated-values"),
+ "xml" | "plist" | "resx" | "xliff" | "xlf" | "csproj" | "vbproj" | "fsproj" | "props"
+ | "targets" => Some("application/xml"),
+ "vtt" => Some("text/vtt"),
+ "srt" => Some("application/x-subrip"),
+ "ics" => Some("text/calendar"),
+ "vcf" => Some("text/vcard"),
+ "eml" => Some("message/rfc822"),
other if crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&other) => {
Some("text/plain")
}
@@ -656,6 +680,18 @@ fn attachment_mime_type(path: &Path) -> Option<&'static str> {
}
}
+fn attachment_payload_mime_type(path: &Path, raw: &[u8]) -> Option<&'static str> {
+ if path
+ .extension()
+ .and_then(|value| value.to_str())
+ .is_some_and(|ext| ext.eq_ignore_ascii_case("3gp"))
+ && is_audio_only_3gp(raw)
+ {
+ return Some("audio/3gpp");
+ }
+ attachment_mime_type(path)
+}
+
// Same shape as the clipboard reader: never traverse a link swapped in after
// the path was validated, and never block the caller on a FIFO.
fn open_attachment_file(path: &Path) -> Result<fs::File, String> {
@@ -701,11 +737,14 @@ fn read_attachment_payload(entry: &NativePathEntry) -> Result<NativeAttachmentFi
let path = &entry.canonical_path;
let mime_type = attachment_mime_type(path)
.ok_or_else(|| "Only chat attachments can be read inline.".to_string())?;
- let is_text_attachment = path
- .extension()
- .and_then(|value| value.to_str())
- .map(|value| value.to_ascii_lowercase())
- .is_some_and(|ext| crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&ext.as_str()));
+ let is_text_attachment = is_text_attachment_name(path)
+ || path
+ .extension()
+ .and_then(|value| value.to_str())
+ .map(|value| value.to_ascii_lowercase())
+ .is_some_and(|ext| {
+ crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&ext.as_str())
+ });
let max_bytes = if is_text_attachment {
MAX_NATIVE_TEXT_BYTES
} else if mime_type.starts_with("image/") {
@@ -742,6 +781,36 @@ fn read_attachment_payload(entry: &NativePathEntry) -> Result<NativeAttachmentFi
if bytes.len() as u64 > max_bytes {
return Err("Attachment is unavailable or too large.".to_string());
}
+ if is_binary_property_list(path, &bytes) {
+ return Err(
+ "Binary property-list files are not supported. Convert the file to text first."
+ .to_string(),
+ );
+ }
+ if is_binary_vobsub(path, &bytes) {
+ return Err("VobSub .sub files are not supported as text attachments.".to_string());
+ }
+ if is_binary_tracker_mod(path, &bytes) {
+ return Err("Tracker .mod audio files are not supported as text attachments.".to_string());
+ }
+ if is_compiled_fortran_mod(path, &bytes) {
+ return Err(
+ "Compiled Fortran .mod modules are not supported as text attachments.".to_string(),
+ );
+ }
+ if is_binary_office_template(path, &bytes) {
+ return Err(
+ "Legacy Word and PowerPoint templates are not supported as text attachments."
+ .to_string(),
+ );
+ }
+ let mime_type = attachment_payload_mime_type(path, &bytes)
+ .ok_or_else(|| "Only chat attachments can be read inline.".to_string())?;
+ // A 3GP path is provisionally video until its track handlers are available.
+ // Reapply the audio cap after an audio-only recording is identified.
+ if mime_type.starts_with("audio/") && bytes.len() as u64 > MAX_NATIVE_ATTACHMENT_BYTES {
+ return Err("Attachment is unavailable or too large.".to_string());
+ }
let name = path
.file_name()
.map(|value| value.to_string_lossy().into_owned())
@@ -824,6 +893,123 @@ mod tests {
}
}
+ fn bmff_box(kind: &[u8; 4], payload: &[u8]) -> Vec<u8> {
+ let size = u32::try_from(8 + payload.len()).unwrap();
+ let mut boxed = Vec::with_capacity(size as usize);
+ boxed.extend_from_slice(&size.to_be_bytes());
+ boxed.extend_from_slice(kind);
+ boxed.extend_from_slice(payload);
+ boxed
+ }
+
+ fn three_gp_with_tracks(handlers: &[[u8; 4]]) -> Vec<u8> {
+ let mut moov_payload = Vec::new();
+ for handler in handlers {
+ let mut hdlr_payload = vec![0; 8];
+ hdlr_payload.extend_from_slice(handler);
+ let mdia = bmff_box(b"mdia", &bmff_box(b"hdlr", &hdlr_payload));
+ moov_payload.extend_from_slice(&bmff_box(b"trak", &mdia));
+ }
+ bmff_box(b"moov", &moov_payload)
+ }
+
+ #[test]
+ fn audio_only_3gp_read_is_stamped_as_audio() {
+ let path = temp_path("recording").with_extension("3gp");
+ let raw = three_gp_with_tracks(&[*b"soun"]);
+ fs::write(&path, &raw).unwrap();
+ let (_state, entry) = attachment_entry(&path);
+ let payload = read_attachment_payload(&entry).unwrap();
+ assert_eq!(payload.mime_type, "audio/3gpp");
+ assert_eq!(BASE64.decode(payload.base64).unwrap(), raw);
+ let _ = fs::remove_file(path);
+ }
+
+ #[test]
+ fn audio_only_3gp_read_reapplies_the_audio_cap() {
+ let path = temp_path("oversized-recording").with_extension("3gp");
+ let mut raw = three_gp_with_tracks(&[*b"soun"]);
+ raw.resize(MAX_NATIVE_ATTACHMENT_BYTES as usize + 1, 0);
+ fs::write(&path, raw).unwrap();
+ let (_state, entry) = attachment_entry(&path);
+ let Err(error) = read_attachment_payload(&entry) else {
+ panic!("expected oversized audio-only 3GP read to fail");
+ };
+ assert!(error.contains("too large"), "unexpected error: {error}");
+ let _ = fs::remove_file(path);
+ }
+
+ #[test]
+ fn any_video_track_keeps_3gp_on_the_video_adapter() {
+ for handlers in [&[*b"vide"][..], &[*b"soun", *b"vide"][..]] {
+ let raw = three_gp_with_tracks(handlers);
+ assert!(!is_audio_only_3gp(&raw));
+ assert_eq!(
+ attachment_payload_mime_type(Path::new("clip.3gp"), &raw),
+ Some("video/3gpp")
+ );
+ }
+ }
+
+ #[test]
+ fn binary_property_list_read_is_rejected() {
+ for extension in ["plist", "strings"] {
+ let path = temp_path("settings").with_extension(extension);
+ fs::write(&path, b"bplist00payload").unwrap();
+ let (_state, entry) = attachment_entry(&path);
+ let Err(error) = read_attachment_payload(&entry) else {
+ panic!("expected binary .{extension} read to fail");
+ };
+ assert!(error.contains("Binary property-list"));
+ let _ = fs::remove_file(path);
+ }
+ }
+
+ #[test]
+ fn binary_vobsub_read_is_rejected() {
+ let path = temp_path("movie").with_extension("sub");
+ fs::write(&path, b"\x00\x00\x01\xbapayload").unwrap();
+ let (_state, entry) = attachment_entry(&path);
+ let Err(error) = read_attachment_payload(&entry) else {
+ panic!("expected binary VobSub read to fail");
+ };
+ assert!(error.contains("VobSub"));
+ let _ = fs::remove_file(path);
+ }
+
+ #[test]
+ fn tracker_mod_read_is_rejected() {
+ let mut tracker = vec![0u8; 1084];
+ tracker[1080..].copy_from_slice(b"M.K.");
+ let mut soundtracker = vec![0u8; 600 + 1024 + 8];
+ soundtracker[43] = 4;
+ soundtracker[45] = 64;
+ soundtracker[470] = 1;
+ soundtracker[471] = 120;
+
+ for (name, bytes) in [("track", tracker), ("classic", soundtracker)] {
+ let path = temp_path(name).with_extension("mod");
+ fs::write(&path, bytes).unwrap();
+ let (_state, entry) = attachment_entry(&path);
+ let Err(error) = read_attachment_payload(&entry) else {
+ panic!("expected tracker MOD read to fail");
+ };
+ assert!(error.contains("Tracker .mod"));
+ let _ = fs::remove_file(path);
+ }
+ }
+
+ #[test]
+ fn extensionless_containerfile_reads_as_text() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("Containerfile");
+ fs::write(&path, b"FROM scratch").unwrap();
+ let (_state, entry) = attachment_entry(&path);
+ let payload = read_attachment_payload(&entry).unwrap();
+ assert_eq!(payload.mime_type, "text/plain");
+ assert_eq!(BASE64.decode(payload.base64).unwrap(), b"FROM scratch");
+ }
+
#[test]
fn image_read_round_trips_and_names_the_file() {
let path = temp_path("photo").with_extension("png");
diff --git a/studio/src-tauri/src/native_path_policy.rs b/studio/src-tauri/src/native_path_policy.rs
index 21114ae1da6..8c48a6adf85 100644
--- a/studio/src-tauri/src/native_path_policy.rs
+++ b/studio/src-tauri/src/native_path_policy.rs
@@ -55,15 +55,43 @@ pub const TRAINING_DATASET_EXTS: &[&str] = &["csv", "json", "jsonl", "parquet"];
/// Keep in sync with `text-attachment-accept.ts`. RAG types are absent so a
/// dropped .txt/.md keeps being indexed.
pub const TEXT_ATTACHMENT_EXTS: &[&str] = &[
+ // Prose and documentation
"text",
"log",
"mdx",
"rst",
+ "adoc",
+ "asciidoc",
+ "org",
+ "textile",
+ "wiki",
+ "tex",
+ "latex",
+ "sty",
+ "cls",
+ "bib",
+ "rmd",
+ "qmd",
+ // Subtitles and captions
+ "srt",
+ "vtt",
+ "sbv",
+ "ass",
+ "ssa",
+ "sub",
+ "lrc",
+ // Structured data
"csv",
"tsv",
+ "psv",
"json",
"jsonl",
"ndjson",
+ "jsonc",
+ "json5",
+ "geojson",
+ "har",
+ "avsc",
"xml",
"yaml",
"yml",
@@ -71,79 +99,492 @@ pub const TEXT_ATTACHMENT_EXTS: &[&str] = &[
"ini",
"cfg",
"conf",
+ "cnf",
"env",
"properties",
+ "plist",
+ "edn",
+ "ron",
+ "cue",
+ "lock",
+ "mod",
+ "sum",
+ "reg",
+ "desktop",
+ "service",
+ // Localisation and interchange
+ "po",
+ "pot",
+ "strings",
+ "resx",
+ "xliff",
+ "xlf",
+ "ics",
+ "vcf",
+ "eml",
+ "mbox",
+ "m3u8",
+ "pls",
+ // Stylesheets and web templates
"css",
"scss",
"sass",
"less",
+ "styl",
"svg",
+ "vue",
+ "svelte",
+ "astro",
+ "pug",
+ "jade",
+ "haml",
+ "slim",
+ "ejs",
+ "erb",
+ "hbs",
+ "handlebars",
+ "mustache",
+ "njk",
+ "jinja",
+ "jinja2",
+ "j2",
+ "twig",
+ "liquid",
+ "cshtml",
+ "razor",
+ "aspx",
+ "jsp",
+ "tpl",
+ "qml",
+ // JavaScript and TypeScript
"js",
"jsx",
"mjs",
"cjs",
"ts",
"tsx",
+ "mts",
+ "cts",
+ // Python
"py",
"pyi",
+ "pyx",
+ "pxd",
"ipynb",
- "rb",
- "php",
- "go",
- "rs",
+ // JVM
"java",
"kt",
"kts",
"scala",
- "swift",
+ "groovy",
+ "gradle",
+ "sbt",
+ "clj",
+ "cljs",
+ "cljc",
+ // Systems languages
"c",
"h",
"cc",
"cpp",
"hpp",
"cxx",
+ "hxx",
+ "hh",
+ "ipp",
+ "inl",
+ "cu",
+ "cuh",
+ "rs",
+ "go",
+ "zig",
+ "odin",
+ "nim",
+ "nims",
+ "nimble",
+ "cr",
+ "d",
+ "v",
+ "sv",
+ "svh",
+ "vhd",
+ "vhdl",
+ "asm",
+ "s",
+ // .NET
"cs",
+ "vb",
+ "vbs",
+ "fs",
+ "fsi",
+ "fsx",
+ "csproj",
+ "vbproj",
+ "fsproj",
+ "sln",
+ "props",
+ "targets",
+ // Apple platforms
"m",
"mm",
+ "swift",
+ "applescript",
+ "metal",
+ // Everything else with a compiler or interpreter
+ "rb",
+ "rake",
+ "gemspec",
+ "podspec",
+ "php",
+ "pl",
+ "pm",
+ "r",
+ "jl",
+ "lua",
+ "tcl",
+ "dart",
+ "hx",
+ "hs",
+ "lhs",
+ "ml",
+ "mli",
+ "ex",
+ "exs",
+ "erl",
+ "hrl",
+ "rkt",
+ "scm",
+ "ss",
+ "lisp",
+ "lsp",
+ "cl",
+ "el",
+ "pas",
+ "pp",
+ "ada",
+ "adb",
+ "ads",
+ "cob",
+ "cbl",
+ "f",
+ "for",
+ "f90",
+ "f95",
+ "f03",
+ "sas",
+ "awk",
+ "sed",
+ "m4",
+ "sol",
+ "move",
+ "cairo",
+ "mojo",
+ "gd",
+ // Shells
"sh",
"bash",
"zsh",
"fish",
+ "ksh",
+ "csh",
+ "tcsh",
+ "nu",
"ps1",
+ "psm1",
+ "psd1",
"bat",
- "lua",
- "pl",
- "pm",
- "r",
- "jl",
- "dart",
- "vue",
- "svelte",
- "astro",
+ "cmd",
+ // Queries and schemas
"sql",
+ "psql",
+ "plsql",
+ "hql",
+ "cql",
"graphql",
"gql",
"proto",
+ "thrift",
+ "capnp",
+ "prisma",
+ // Infrastructure and build
"tf",
"tfvars",
- "gradle",
+ "tfstate",
+ "hcl",
+ "nix",
+ "dhall",
+ "bicep",
"dockerfile",
+ "containerfile",
"makefile",
+ "mk",
+ "mak",
"cmake",
+ "ninja",
+ "bzl",
+ "bazel",
+ "star",
+ "starlark",
+ "gn",
+ "gni",
+ "pro",
+ "pri",
+ "cabal",
+ "opam",
+ // Shaders
+ "glsl",
+ "frag",
+ "vert",
+ "geom",
+ "comp",
+ "hlsl",
+ "wgsl",
+ "shader",
+ // Diagrams, specs and request files
+ "mmd",
+ "mermaid",
+ "puml",
+ "plantuml",
+ "dot",
+ "gv",
+ "feature",
+ "robot",
+ "http",
+ "rest",
+ // Diffs
"diff",
"patch",
];
+/// Conventional extensionless names represented by dotted frontend accept tokens.
+pub const TEXT_ATTACHMENT_NAMES: &[&str] = &["containerfile", "dockerfile", "makefile"];
+
+pub fn is_text_attachment_name(path: &Path) -> bool {
+ path.file_name()
+ .and_then(|value| value.to_str())
+ .map(|value| value.to_ascii_lowercase())
+ .is_some_and(|name| TEXT_ATTACHMENT_NAMES.contains(&name.as_str()))
+}
+
+pub fn is_binary_property_list(path: &Path, bytes: &[u8]) -> bool {
+ (has_extension(path, "plist") || has_extension(path, "strings"))
+ && bytes.starts_with(b"bplist00")
+}
+
+pub fn is_binary_vobsub(path: &Path, bytes: &[u8]) -> bool {
+ has_extension(path, "sub") && bytes.starts_with(b"\x00\x00\x01\xba")
+}
+
+const TRACKER_MOD_MAGICS: &[&[u8; 4]] = &[
+ b"M.K.", b"M!K!", b"!PM!", b"PATT", b"NSMS", b"LARD", b"M&K!", b"FEST", b"N.T.", b"OKTA",
+ b"OCTA", b"CD81", b"CD61", b"FLT4", b"FLT8", b"EXO4", b"EXO8", b".M.K", b"WARD", b"M\0\0\0",
+ b"8\0\0\0",
+];
+
+const SOUNDTRACKER_HEADER_BYTES: usize = 600;
+const SOUNDTRACKER_PATTERN_BYTES: usize = 1024;
+const SOUNDTRACKER_MAX_PATTERN_ERRORS: usize = 22;
+const SOUNDTRACKER_PERIODS: &[u16] = &[
+ 856, 808, 762, 720, 678, 640, 604, 570, 538, 508, 480, 453, 428, 404, 381, 360, 339, 320, 302,
+ 285, 269, 254, 240, 226, 214, 202, 190, 180, 170, 160, 151, 143, 135, 127, 120, 113, 763, 679,
+ 641, 571, 539, 509, 429, 340, 321, 300, 286, 270, 227, 191, 162,
+];
+
+fn big_endian_word(bytes: &[u8], offset: usize) -> usize {
+ usize::from(bytes[offset]) * 256 + usize::from(bytes[offset + 1])
+}
+
+fn soundtracker_pattern_end(bytes: &[u8]) -> Option<usize> {
+ if bytes.len() < SOUNDTRACKER_HEADER_BYTES {
+ return None;
+ }
+
+ let mut sample_bytes = 0usize;
+ for sample in 0..15 {
+ let offset = 20 + sample * 30;
+ let length_words = big_endian_word(bytes, offset + 22);
+ let finetune = bytes[offset + 24];
+ let volume = bytes[offset + 25];
+ let loop_start = big_endian_word(bytes, offset + 26);
+ let loop_length = big_endian_word(bytes, offset + 28);
+ if volume > 0x40
+ || finetune & 0xf0 != 0
+ || length_words > 0x8000
+ || loop_length > 0x8000
+ || (loop_start >> 1) > length_words
+ || (length_words > 0 && (loop_start >> 1) == length_words)
+ || (length_words == 0 && loop_start > 0)
+ {
+ return None;
+ }
+ sample_bytes = sample_bytes.checked_add(length_words * 2)?;
+ }
+ if sample_bytes < 8 {
+ return None;
+ }
+
+ let song_length = usize::from(bytes[470]);
+ if song_length == 0 || song_length > 128 {
+ return None;
+ }
+ let mut max_pattern = 0usize;
+ let mut max_used_pattern = 0usize;
+ for (index, pattern) in bytes[472..600].iter().copied().enumerate() {
+ let pattern = usize::from(pattern);
+ if pattern > 0x7f {
+ return None;
+ }
+ max_pattern = max_pattern.max(pattern);
+ if index < song_length {
+ max_used_pattern = max_used_pattern.max(pattern);
+ }
+ }
+
+ let mut pattern_count = max_pattern + 1;
+ let used_pattern_count = max_used_pattern + 1;
+ let mut expected_size = SOUNDTRACKER_HEADER_BYTES
+ .checked_add(pattern_count.checked_mul(SOUNDTRACKER_PATTERN_BYTES)?)?
+ .checked_add(sample_bytes)?;
+ let used_expected_size = SOUNDTRACKER_HEADER_BYTES
+ .checked_add(used_pattern_count.checked_mul(SOUNDTRACKER_PATTERN_BYTES)?)?
+ .checked_add(sample_bytes)?;
+ if bytes.len() < expected_size && bytes.len() == used_expected_size {
+ pattern_count = used_pattern_count;
+ expected_size = used_expected_size;
+ }
+ let pattern_end = SOUNDTRACKER_HEADER_BYTES
+ .checked_add(pattern_count.checked_mul(SOUNDTRACKER_PATTERN_BYTES)?)?;
+ (bytes.len() >= expected_size.saturating_mul(93) / 100 && bytes.len() >= pattern_end)
+ .then_some(pattern_end)
+}
+
+fn has_soundtracker_pattern_data(bytes: &[u8], pattern_end: usize) -> bool {
+ let Some(patterns) = bytes.get(SOUNDTRACKER_HEADER_BYTES..pattern_end) else {
+ return false;
+ };
+ let mut errors = 0usize;
+ for event in patterns.chunks_exact(4) {
+ let sample = (event[0] & 0xf0) | (event[2] >> 4);
+ let period = (u16::from(event[0] & 0x0f) << 8) | u16::from(event[1]);
+ if sample > 15 {
+ errors += 1;
+ if errors > SOUNDTRACKER_MAX_PATTERN_ERRORS {
+ return false;
+ }
+ }
+ if period != 0 && !SOUNDTRACKER_PERIODS.contains(&period) {
+ errors += 1;
+ if errors > SOUNDTRACKER_MAX_PATTERN_ERRORS {
+ return false;
+ }
+ }
+ }
+ true
+}
+
+/// Legacy Word `.dot` and PowerPoint `.pot` templates are OLE compound files,
+/// unlike Graphviz `.dot` and gettext `.pot`.
+pub fn is_binary_office_template(path: &Path, bytes: &[u8]) -> bool {
+ (has_extension(path, "dot") || has_extension(path, "pot"))
+ && bytes.starts_with(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1")
+}
+
+/// gfortran writes a compiled `.mod` module as gzip; text `go.mod` never is.
+pub fn is_compiled_fortran_mod(path: &Path, bytes: &[u8]) -> bool {
+ has_extension(path, "mod") && bytes.starts_with(b"\x1f\x8b")
+}
+
+/// Detect marker-bearing MODs and earlier 15-sample Soundtracker modules.
+pub fn is_binary_tracker_mod(path: &Path, bytes: &[u8]) -> bool {
+ if !has_extension(path, "mod") {
+ return false;
+ }
+ if let Some(marker) = bytes.get(1080..1084) {
+ if TRACKER_MOD_MAGICS
+ .iter()
+ .any(|candidate| marker == candidate.as_slice())
+ || (marker[0].is_ascii_digit() && marker[0] != b'0' && &marker[1..] == b"CHN")
+ || (marker[0].is_ascii_digit()
+ && marker[0] != b'0'
+ && marker[1].is_ascii_digit()
+ && (&marker[2..] == b"CH" || &marker[2..] == b"CN"))
+ || (&marker[..3] == b"TDZ" && marker[3].is_ascii_digit() && marker[3] != b'0')
+ || (&marker[..3] == b"FA0" && (b'4'..=b'8').contains(&marker[3]))
+ {
+ return true;
+ }
+ }
+ soundtracker_pattern_end(bytes)
+ .is_some_and(|pattern_end| has_soundtracker_pattern_data(bytes, pattern_end))
+}
+
+fn bmff_box_payloads<'a>(data: &'a [u8], wanted: &[u8; 4]) -> Vec<&'a [u8]> {
+ let mut payloads = Vec::new();
+ let mut offset = 0usize;
+ while data.len().saturating_sub(offset) >= 8 {
+ let size32 = u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap());
+ let box_type = &data[offset + 4..offset + 8];
+ let (header_size, box_size) = match size32 {
+ 0 => (8usize, data.len() - offset),
+ 1 if data.len().saturating_sub(offset) >= 16 => {
+ let size64 = u64::from_be_bytes(data[offset + 8..offset + 16].try_into().unwrap());
+ let Ok(size) = usize::try_from(size64) else {
+ break;
+ };
+ (16, size)
+ }
+ 1 => break,
+ size => (8, size as usize),
+ };
+ if box_size < header_size || box_size > data.len() - offset {
+ break;
+ }
+ if box_type == wanted {
+ payloads.push(&data[offset + header_size..offset + box_size]);
+ }
+ offset += box_size;
+ }
+ payloads
+}
+
+/// Whether a 3GP container has audio tracks and no video tracks.
+pub fn is_audio_only_3gp(raw: &[u8]) -> bool {
+ let mut has_audio = false;
+ let mut has_video = false;
+ for moov in bmff_box_payloads(raw, b"moov") {
+ for trak in bmff_box_payloads(moov, b"trak") {
+ for mdia in bmff_box_payloads(trak, b"mdia") {
+ for hdlr in bmff_box_payloads(mdia, b"hdlr") {
+ if hdlr.len() < 12 {
+ continue;
+ }
+ match &hdlr[8..12] {
+ b"soun" => has_audio = true,
+ b"vide" => has_video = true,
+ _ => {}
+ }
+ }
+ }
+ }
+ }
+ has_audio && !has_video
+}
+
/// Vision chat image attachments; keep in sync with `drop-paths.ts` `CHAT_IMAGE_DROP_ACCEPT`.
pub const IMAGE_ATTACHMENT_EXTS: &[&str] = &["jpg", "jpeg", "png", "webp", "gif"];
/// Chat audio attachments; keep in sync with `audio-attachment-adapter.ts` `accept`.
-pub const AUDIO_ATTACHMENT_EXTS: &[&str] = &["wav", "mp3", "m4a", "ogg", "oga", "flac"];
+pub const AUDIO_ATTACHMENT_EXTS: &[&str] = &[
+ "wav", "mp3", "m4a", "ogg", "oga", "opus", "flac", "aac", "aiff", "aif", "aifc", "caf", "wma",
+ "amr", "mp2",
+];
/// Chat video attachments; keep in sync with `drop-paths.ts`
/// `CHAT_VIDEO_DROP_ACCEPT`. llama-server decodes with ffmpeg, so this is what
/// ffmpeg reads, not what the webview can play.
-pub const VIDEO_ATTACHMENT_EXTS: &[&str] = &["mp4", "mov", "webm", "mkv", "avi"];
+pub const VIDEO_ATTACHMENT_EXTS: &[&str] = &[
+ "mp4", "m4v", "mov", "webm", "mkv", "avi", "mpg", "mpeg", "wmv", "flv", "3gp", "ogv",
+];
fn accepted_attachment_exts() -> impl Iterator<Item = &'static &'static str> {
ATTACHMENT_EXTS
@@ -160,8 +601,8 @@ pub fn classify_native_attachment_path(path: &Path) -> Result<ClassifiedPath, St
if classified.path_type != NativePathType::File {
return Err("Only files can be attached to a chat.".to_string());
}
- let supported =
- accepted_attachment_exts().any(|ext| has_extension(&classified.canonical_path, ext));
+ let supported = is_text_attachment_name(&classified.canonical_path)
+ || accepted_attachment_exts().any(|ext| has_extension(&classified.canonical_path, ext));
if !supported {
return Err(format!(
"Unsupported attachment type. Supported: {}",
@@ -789,6 +1230,128 @@ mod tests {
let _ = fs::remove_file(path);
}
+ #[test]
+ fn extensionless_containerfile_is_an_attachment() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("Containerfile");
+ fs::write(&path, b"FROM scratch").unwrap();
+
+ let classified = classify_native_attachment_path(&path).unwrap();
+ assert_eq!(classified.path_kind, NativePathKind::Attachment);
+ assert!(classified
+ .allowed_operations
+ .contains(&NativePathOperation::Attach));
+ }
+
+ #[test]
+ fn binary_property_list_magic_is_detected() {
+ assert!(is_binary_property_list(
+ Path::new("settings.plist"),
+ b"bplist00payload"
+ ));
+ assert!(!is_binary_property_list(
+ Path::new("settings.plist"),
+ b"<?xml version=\"1.0\"?>"
+ ));
+ assert!(!is_binary_property_list(
+ Path::new("settings.txt"),
+ b"bplist00payload"
+ ));
+ assert!(is_binary_property_list(
+ Path::new("Localizable.strings"),
+ b"bplist00payload"
+ ));
+ assert!(!is_binary_property_list(
+ Path::new("Localizable.strings"),
+ b"\"hello\" = \"world\";"
+ ));
+ }
+
+ #[test]
+ fn binary_vobsub_magic_is_detected() {
+ assert!(is_binary_vobsub(
+ Path::new("movie.sub"),
+ b"\x00\x00\x01\xbapayload"
+ ));
+ assert!(!is_binary_vobsub(
+ Path::new("movie.sub"),
+ b"{1}{24}Subtitle"
+ ));
+ assert!(!is_binary_vobsub(
+ Path::new("movie.txt"),
+ b"\x00\x00\x01\xbapayload"
+ ));
+ }
+
+ #[test]
+ fn legacy_office_templates_are_detected_without_rejecting_text_templates() {
+ // OLE compound file header, shared by legacy .dot and .pot templates.
+ let ole = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1payload";
+ assert!(is_binary_office_template(Path::new("report.dot"), ole));
+ assert!(is_binary_office_template(Path::new("deck.pot"), ole));
+ // Graphviz and gettext keep the same extensions and stay accepted.
+ assert!(!is_binary_office_template(
+ Path::new("graph.dot"),
+ b"digraph G { a -> b; }"
+ ));
+ assert!(!is_binary_office_template(
+ Path::new("messages.pot"),
+ b"msgid \"\"\nmsgstr \"\"\n"
+ ));
+ assert!(!is_binary_office_template(Path::new("deck.ppt"), ole));
+ }
+
+ #[test]
+ fn compiled_fortran_modules_are_detected_without_rejecting_go_mod() {
+ // gfortran gzips its module files; the tracker check reads a music header
+ // and never sees these.
+ assert!(is_compiled_fortran_mod(
+ Path::new("kinds.mod"),
+ b"\x1f\x8b\x08\x00GFORTRAN module"
+ ));
+ assert!(!is_compiled_fortran_mod(
+ Path::new("go.mod"),
+ b"module example.com/app\n\ngo 1.22\n"
+ ));
+ // An uncompressed gfortran module is readable text and stays accepted.
+ assert!(!is_compiled_fortran_mod(
+ Path::new("kinds.mod"),
+ b"GFORTRAN module version '15'\n"
+ ));
+ assert!(!is_compiled_fortran_mod(
+ Path::new("archive.gz"),
+ b"\x1f\x8b\x08\x00"
+ ));
+ }
+
+ #[test]
+ fn tracker_mod_magic_is_detected_without_rejecting_text_modules() {
+ let mut tracker = vec![0u8; 1084];
+ tracker[1080..].copy_from_slice(b"M.K.");
+ assert!(is_binary_tracker_mod(Path::new("track.mod"), &tracker));
+
+ let mut soundtracker = vec![0u8; 600 + 1024 + 8];
+ soundtracker[43] = 4; // Four big-endian words of sample data.
+ soundtracker[45] = 64;
+ soundtracker[470] = 1;
+ soundtracker[471] = 120;
+ assert!(is_binary_tracker_mod(
+ Path::new("classic.mod"),
+ &soundtracker
+ ));
+
+ let mut go_mod = b"module example.com/project\n".to_vec();
+ go_mod.resize(1200, b' ');
+ assert!(!is_binary_tracker_mod(Path::new("go.mod"), &go_mod));
+ assert!(!is_binary_tracker_mod(Path::new("track.bin"), &tracker));
+ }
+
+ #[test]
+ fn legacy_m3u_is_not_advertised_as_utf8_text() {
+ assert!(!TEXT_ATTACHMENT_EXTS.contains(&"m3u"));
+ assert!(TEXT_ATTACHMENT_EXTS.contains(&"m3u8"));
+ }
+
// The composer takes audio uploads, so a dropped one has to classify too.
#[test]
fn audio_attachments_are_accepted() {