:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! A Rust `String` cannot hold a lone surrogate, so `JsString` stores WTF-8, where a lone surrogate takes its 3-byte form.
use std::borrow::Cow;
use std::fmt;
#[derive(Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)]
pub struct JsString {
/// WTF-8. Valid UTF-8 exactly when `lone_surrogates` is 0.
bytes: Vec<u8>,
/// The number of lone surrogates in `bytes`, so a joined pair needs no rescan.
lone_surrogates: usize,
}
impl JsString {
pub fn new() -> Self {
Self::default()
}
pub fn with_capacity(capacity: usize) -> Self {
Self {
bytes: Vec::with_capacity(capacity),
lone_surrogates: 0,
}
}
pub fn as_str(&self) -> Option<&str> {
if self.lone_surrogates > 0 {
None
} else {
// SAFETY: without a lone surrogate the bytes are valid UTF-8 by construction, and a checked conversion would rescan.
// nosemgrep: rust.lang.security.unsafe-usage.unsafe-usage
Some(unsafe { std::str::from_utf8_unchecked(&self.bytes) })
}
}
/// The text as `Buffer.from(str)` would encode it: each lone surrogate becomes U+FFFD.
pub fn to_utf8_lossy(&self) -> Cow<'_, str> {
match self.as_str() {
Some(s) => Cow::Borrowed(s),
None => {
let mut out = String::with_capacity(self.bytes.len());
for unit in self.chars_or_surrogates() {
match unit {
CharOrSurrogate::Char(c) => out.push(c),
CharOrSurrogate::Surrogate(_) => out.push('\u{FFFD}'),
}
}
Cow::Owned(out)
}
}
}
pub fn into_string_lossy(self) -> String {
if self.lone_surrogates > 0 {
self.to_utf8_lossy().into_owned()
} else {
// SAFETY: as in `as_str`.
// nosemgrep: rust.lang.security.unsafe-usage.unsafe-usage
unsafe { String::from_utf8_unchecked(self.bytes) }
}
}
pub fn push_str(&mut self, s: &str) {
self.bytes.extend_from_slice(s.as_bytes());
}
pub fn push(&mut self, c: char) {
let mut buf = [0u8; 4];
self.bytes
.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
}
pub fn push_js(&mut self, other: &JsString) {
// A join can happen only at the seam, so the bytes copy unless the seam joins.
if other.lone_surrogates == 0
|| !other.starts_with_low_surrogate()
|| self.trailing_high_surrogate().is_none()
{
self.bytes.extend_from_slice(&other.bytes);
self.lone_surrogates += other.lone_surrogates;
return;
}
for unit in other.chars_or_surrogates() {
match unit {
CharOrSurrogate::Char(c) => self.push(c),
CharOrSurrogate::Surrogate(s) => self.push_code_unit(s),
}
}
}
/// A low surrogate after a lone high surrogate joins it into one code point, as JavaScript concatenation does.
pub fn push_code_unit(&mut self, unit: u16) {
if !(0xD800..=0xDFFF).contains(&unit) {
self.push(char::from_u32(unit as u32).expect("non-surrogate unit is a char"));
return;
}
if (0xDC00..=0xDFFF).contains(&unit) {
if let Some(high) = self.trailing_high_surrogate() {
self.bytes.truncate(self.bytes.len() - 3);
let cp = 0x10000 + (((high as u32) - 0xD800) << 10) + ((unit as u32) - 0xDC00);
self.push(char::from_u32(cp).expect("surrogate pair is a char"));
self.lone_surrogates -= 1;
return;
}
}
self.bytes.push(0xE0 | ((unit >> 12) as u8));
self.bytes.push(0x80 | (((unit >> 6) & 0x3F) as u8));
self.bytes.push(0x80 | ((unit & 0x3F) as u8));
self.lone_surrogates += 1;
}
fn trailing_high_surrogate(&self) -> Option<u16> {
let n = self.bytes.len();
if self.lone_surrogates == 0 || n < 3 {
return None;
}
let b = &self.bytes[n - 3..];
if b[0] == 0xED && (0xA0..=0xAF).contains(&b[1]) {
Some(0xD000 | (((b[1] & 0x3F) as u16) << 6) | ((b[2] & 0x3F) as u16))
} else {
None
}
}
fn starts_with_low_surrogate(&self) -> bool {
matches!(self.bytes.as_slice(), [0xED, 0xB0..=0xBF, ..])
}
/// Cuts back to `len` bytes, where `lone_surrogates` is the count that the prefix had.
pub(crate) fn truncate_to(&mut self, len: usize, lone_surrogates: usize) {
self.bytes.truncate(len);
self.lone_surrogates = lone_surrogates;
}
pub(crate) fn lone_surrogate_count(&self) -> usize {
self.lone_surrogates
}
pub fn utf16_len(&self) -> usize {
match self.as_str() {
Some(s) => utf16_len(s),
None => self
.chars_or_surrogates()
.map(|u| match u {
CharOrSurrogate::Char(c) => c.len_utf16(),
CharOrSurrogate::Surrogate(_) => 1,
})
.sum(),
}
}
pub fn utf16_slice(&self, start: usize, end: usize) -> JsString {
let mut out = JsString::new();
let mut pos = 0usize;
for unit in self.chars_or_surrogates() {
if pos >= end {
break;
}
match unit {
CharOrSurrogate::Char(c) => {
let mut buf = [0u16; 2];
for &u in c.encode_utf16(&mut buf).iter() {
if pos >= start && pos < end {
out.push_code_unit(u);
}
pos += 1;
}
}
CharOrSurrogate::Surrogate(s) => {
if pos >= start && pos < end {
out.push_code_unit(s);
}
pos += 1;
}
}
}
out
}
pub fn bytes_wtf8(&self) -> &[u8] {
&self.bytes
}
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
pub fn chars_or_surrogates(&self) -> impl Iterator<Item = CharOrSurrogate> + '_ {
Wtf8Iter {
bytes: &self.bytes,
pos: 0,
}
}
pub fn utf8_byte_len(&self) -> usize {
// WTF-8 encodes a lone surrogate in three bytes, the same as U+FFFD.
self.bytes.len()
}
}
pub enum CharOrSurrogate {
Char(char),
Surrogate(u16),
}
impl CharOrSurrogate {
pub fn as_char(&self) -> Result<char, u16> {
match self {
CharOrSurrogate::Char(c) => Ok(*c),
CharOrSurrogate::Surrogate(s) => Err(*s),
}
}
}
struct Wtf8Iter<'a> {
bytes: &'a [u8],
pos: usize,
}
impl Iterator for Wtf8Iter<'_> {
type Item = CharOrSurrogate;
fn next(&mut self) -> Option<CharOrSurrogate> {
let b = self.bytes;
let i = self.pos;
if i >= b.len() {
return None;
}
let first = b[i];
let (cp, len) = if first < 0x80 {
(first as u32, 1)
} else if first < 0xE0 {
(
(((first & 0x1F) as u32) << 6) | ((b[i + 1] & 0x3F) as u32),
2,
)
} else if first < 0xF0 {
(
(((first & 0x0F) as u32) << 12)
| (((b[i + 1] & 0x3F) as u32) << 6)
| ((b[i + 2] & 0x3F) as u32),
3,
)
} else {
(
(((first & 0x07) as u32) << 18)
| (((b[i + 1] & 0x3F) as u32) << 12)
| (((b[i + 2] & 0x3F) as u32) << 6)
| ((b[i + 3] & 0x3F) as u32),
4,
)
};
self.pos += len;
Some(match char::from_u32(cp) {
Some(c) => CharOrSurrogate::Char(c),
None => CharOrSurrogate::Surrogate(cp as u16),
})
}
}
impl From<&str> for JsString {
fn from(s: &str) -> Self {
Self {
bytes: s.as_bytes().to_vec(),
lone_surrogates: 0,
}
}
}
impl From<String> for JsString {
fn from(s: String) -> Self {
Self {
bytes: s.into_bytes(),
lone_surrogates: 0,
}
}
}
impl fmt::Debug for JsString {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.to_utf8_lossy(), f)
}
}
impl fmt::Display for JsString {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.to_utf8_lossy())
}
}
impl PartialEq<str> for JsString {
fn eq(&self, other: &str) -> bool {
self.as_str() == Some(other)
}
}
impl PartialEq<&str> for JsString {
fn eq(&self, other: &&str) -> bool {
self.as_str() == Some(*other)
}
}
pub fn utf16_len(s: &str) -> usize {
if s.is_ascii() {
return s.len();
}
s.chars().map(char::len_utf16).sum()
}
/// Use it only where the slice goes straight to bytes. Use `JsString::utf16_slice` where it can reach `JSON.stringify`.
pub fn utf16_slice_lossy(s: &str, start: usize, end: usize) -> String {
if s.is_ascii() {
let end = end.min(s.len());
let start = start.min(end);
return s[start..end].to_string();
}
JsString::from(s)
.utf16_slice(start, end)
.into_string_lossy()
}
/// `None` when `s` has at most `max` UTF-16 units. It reads only `max + 1` units, so a large body costs no more than the cap.
pub fn utf16_prefix_lossy(s: &str, max: usize) -> Option<String> {
if s.is_ascii() {
return (s.len() > max).then(|| s[..max].to_string());
}
let mut units = 0usize;
for (at, c) in s.char_indices() {
let width = c.len_utf16();
if units + width > max {
let mut prefix = s[..at].to_string();
if units < max {
// The cut falls inside a surrogate pair. The lone high surrogate converts to U+FFFD.
prefix.push('\u{FFFD}');
}
return Some(prefix);
}
units += width;
}
None
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn slicing_an_emoji_leaves_a_lone_surrogate() {
let s = JsString::from("a🚀b");
assert_eq!(s.utf16_len(), 4);
let cut = s.utf16_slice(0, 2);
assert_eq!(cut.utf16_len(), 2);
assert!(cut.as_str().is_none());
assert_eq!(cut.to_utf8_lossy(), "a\u{FFFD}");
assert_eq!(utf16_slice_lossy("a🚀b", 0, 2), "a\u{FFFD}");
assert_eq!(utf16_prefix_lossy("a🚀b", 2).as_deref(), Some("a\u{FFFD}"));
assert_eq!(utf16_prefix_lossy("a🚀b", 3).as_deref(), Some("a🚀"));
assert_eq!(utf16_prefix_lossy("a🚀b", 4), None);
}
#[test]
fn a_split_pair_rejoins_on_concatenation() {
let s = JsString::from("🚀");
let mut joined = s.utf16_slice(0, 1);
joined.push_js(&s.utf16_slice(1, 2));
assert_eq!(joined, JsString::from("🚀"));
assert!(joined.as_str().is_some());
let mut mixed = JsString::new();
mixed.push_code_unit(0xD800);
mixed.push_code_unit(0xD83D);
mixed.push_code_unit(0xDE80);
assert!(mixed.as_str().is_none(), "the first lone surrogate remains");
assert_eq!(mixed.to_utf8_lossy(), "\u{FFFD}🚀");
}
}