:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Person-property size enforcement, shared by the leader (admission-time
//! trimming and rejection) and anything that needs to reason about the
//! Postgres row the writer will eventually produce.
//!
//! The authority on "too big" is the `check_properties_size` constraint on
//! `posthog_person`: `pg_column_size(properties) <= 655360`. That measures
//! the JSONB *binary encoding*, not the raw JSON text — per-element entry
//! headers, PG's base-10000 numeric format, and alignment padding all
//! count. `jsonb_column_size` reproduces that number exactly so admission
//! enforces the real constraint rather than a proxy with a hoped-for
//! margin; a property test compares it against a live Postgres
//! (`pg_column_size($1::jsonb)`) so any drift from PG's encoding fails
//! loudly. The JSONB on-disk format is stable across PG versions by
//! design (pg_upgrade compatibility), which is what makes reimplementing
//! it sound.
use std::collections::HashSet;
use std::sync::LazyLock;
use serde_json::Value;
/// Properties that must never be trimmed, matching the Node.js ingestion
/// pipeline's `ALL_PROTECTED_PROPERTIES` in `person-property-utils.ts`.
static PROTECTED_PROPERTIES: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
HashSet::from([
// Core person properties
"email",
"name",
// Event-to-person properties (mobile)
"$app_build",
"$app_name",
"$app_namespace",
"$app_version",
// Event-to-person properties (web)
"$browser",
"$browser_version",
"$device_type",
"$current_url",
"$pathname",
"$os",
"$os_name",
"$os_version",
"$referring_domain",
"$referrer",
"$screen_height",
"$screen_width",
"$viewport_height",
"$viewport_width",
"$raw_user_agent",
// UTM and campaign tracking
"utm_source",
"utm_medium",
"utm_campaign",
"utm_content",
"utm_name",
"utm_term",
"gclid",
"gad_source",
"gclsrc",
"dclid",
"gbraid",
"wbraid",
"fbclid",
"msclkid",
"twclid",
"li_fat_id",
"mc_cid",
"igshid",
"ttclid",
"rdt_cid",
"irclid",
"_kx",
"epik",
"qclid",
"sccid",
"oppref",
"bfcid",
"$fbc",
// Session and page tracking
"$session_id",
"$window_id",
"$pageview_id",
"$host",
// Identity and device tracking
"$user_id",
"$device_id",
"$anon_distinct_id",
// Initial/first-touch properties
"$initial_referrer",
"$initial_referring_domain",
"$initial_utm_source",
"$initial_utm_medium",
"$initial_utm_campaign",
"$initial_utm_content",
"$initial_utm_term",
])
});
/// Returns true if the property can be trimmed (is not protected).
pub fn can_trim_property(name: &str) -> bool {
!PROTECTED_PROPERTIES.contains(name)
}
/// Floats at or above this magnitude are clamped by [`sanitize_for_jsonb`].
/// Postgres renders jsonb numerics in expanded decimal notation (never
/// e-notation), and serde_json's parser overflows on integer expansions of
/// ~1e308 and beyond — so a number in the window [1e308, f64::MAX] applies
/// to PG but can never be read back by a Rust consumer (including the
/// leader's own PG fallback load). 1e307 expands to 308 digits, which
/// parses back exactly.
pub const MAX_JSONB_SAFE_MAGNITUDE: f64 = 1e307;
/// What [`sanitize_for_jsonb`] rewrote, for the caller's counters.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct SanitizeStats {
/// Strings (keys or values) that contained `\u{0000}`.
pub nul_strings: u64,
/// Floats clamped to ±[`MAX_JSONB_SAFE_MAGNITUDE`].
pub clamped_numbers: u64,
}
impl SanitizeStats {
pub fn changed(&self) -> bool {
self.nul_strings > 0 || self.clamped_numbers > 0
}
}
/// Rewrite the merged state so that it both applies to Postgres jsonb and
/// round-trips back out of it *unchanged* — a sanitized value compares
/// equal to its own jsonb round-trip, which idempotency guards (frozen
/// request rows, no-op detection against cached state) rely on. At any
/// depth:
///
/// * `\u{0000}` (NUL) becomes `\u{FFFD}` in every string — keys and
/// values — mirroring the Node.js pipeline's `sanitizeJsonbValue`;
/// Postgres refuses NUL outright. A sanitized key can collide with an
/// existing key; the later entry wins, matching what Postgres itself
/// does when parsing JSON text with duplicate keys.
/// * Floats beyond ±[`MAX_JSONB_SAFE_MAGNITUDE`] are clamped to it, so
/// Postgres's expanded numeric rendering stays parseable on the way
/// back (see the constant's doc).
/// * Floats holding an exact integer become JSON integers: jsonb renders
/// numerics in plain decimal, so `1e17` comes back as
/// `100000000000000000`, which serde_json parses as an integer — never
/// equal to the float that went in. Converting up front makes the
/// stored form and the sanitized form the same value.
pub fn sanitize_for_jsonb(value: &mut Value) -> SanitizeStats {
let mut stats = SanitizeStats::default();
sanitize_value(value, &mut stats);
stats
}
fn sanitize_value(value: &mut Value, stats: &mut SanitizeStats) {
match value {
Value::String(s) => {
if s.contains('\u{0000}') {
*s = s.replace('\u{0000}', "\u{FFFD}");
stats.nul_strings += 1;
}
}
Value::Number(n) => {
// Only floats can exceed the safe magnitude: i64/u64 top out
// around 1.8e19.
if n.is_f64() {
if let Some(f) = n.as_f64() {
if f.abs() > MAX_JSONB_SAFE_MAGNITUDE {
let clamped = MAX_JSONB_SAFE_MAGNITUDE.copysign(f);
*value = Value::from(clamped);
stats.clamped_numbers += 1;
} else if let Some(int) = integral_float_as_int(f) {
*value = int;
}
}
}
}
Value::Array(items) => {
for item in items {
sanitize_value(item, stats);
}
}
Value::Object(map) => {
if map.keys().any(|k| k.contains('\u{0000}')) {
let entries: Vec<(String, Value)> = std::mem::take(map).into_iter().collect();
for (key, mut val) in entries {
sanitize_value(&mut val, stats);
let key = if key.contains('\u{0000}') {
stats.nul_strings += 1;
key.replace('\u{0000}', "\u{FFFD}")
} else {
key
};
map.insert(key, val);
}
} else {
for val in map.values_mut() {
sanitize_value(val, stats);
}
}
}
_ => {}
}
}
/// The integer a float canonicalizes to when its value is one, i.e. when
/// Postgres would hand the number back as an integer literal. Positive
/// values take the u64 path because that is what serde_json's parser
/// yields for a positive integer literal.
fn integral_float_as_int(f: f64) -> Option<Value> {
if f.fract() != 0.0 {
return None;
}
if f >= 0.0 {
// `as u64` saturates: exactly 2^64 aliases to u64::MAX, whose own
// f64 round-trip is 2^64 again — the equality check would convert
// to the wrong integer. u64::MAX itself is not representable as
// f64, so excluding the saturation point loses nothing.
let u = f as u64;
(u != u64::MAX && u as f64 == f).then(|| Value::from(u))
} else {
// No aliasing on this side: i64::MIN (-2^63) is exactly
// representable, and anything below it saturates to i64::MIN,
// whose round-trip then fails the equality check.
let i = f as i64;
(i as f64 == f).then(|| Value::from(i))
}
}
/// Rewrite number tokens that serde_json refuses so the document parses,
/// mimicking `JSON.parse`'s rounding. Postgres renders jsonb numerics in
/// expanded decimal (never e-notation), and serde_json rejects expansions
/// near `f64::MAX` even when the value itself is representable — while
/// every JS consumer reads the same bytes fine, because `JSON.parse`
/// rounds to the nearest double instead of erroring. Legacy and
/// Node-written rows can therefore hold numerics the leader's strict
/// parser chokes on.
///
/// Tokens serde_json accepts are copied verbatim (no precision change);
/// rejected tokens whose value fits a double are rewritten to its
/// shortest representation — the exact value `JSON.parse` yields — and
/// magnitudes beyond `f64::MAX` (unreachable from JS; Python bigints and
/// raw SQL) are clamped to ±[`MAX_JSONB_SAFE_MAGNITUDE`], where today's
/// pipeline reads `Infinity` and rewrites it as `null`. String content is
/// copied verbatim, so digits inside strings are never touched. Intended
/// for the rare failure path only: callers parse normally first and
/// rewrite only when that fails.
pub fn rewrite_out_of_range_numbers(text: &str) -> String {
let bytes = text.as_bytes();
let mut out = String::with_capacity(text.len());
let mut i = 0;
while i < bytes.len() {
match bytes[i] {
b'"' => {
// Copy the whole string token verbatim, escape-aware.
let start = i;
i += 1;
while i < bytes.len() {
match bytes[i] {
b'\\' => i = (i + 2).min(bytes.len()),
b'"' => {
i += 1;
break;
}
_ => i += 1,
}
}
out.push_str(&text[start..i]);
}
b'-' | b'0'..=b'9' => {
let start = i;
while i < bytes.len()
&& matches!(bytes[i], b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-')
{
i += 1;
}
let token = &text[start..i];
if token.parse::<serde_json::Number>().is_ok() {
out.push_str(token);
} else {
match token.parse::<f64>() {
// std's parser rounds to nearest like JSON.parse,
// so a finite result is the double JS reads.
Ok(f) if f.is_finite() => out.push_str(
&serde_json::Number::from_f64(f)
.expect("finite float converts")
.to_string(),
),
_ => {
let sign = if token.starts_with('-') { "-" } else { "" };
out.push_str(&format!("{sign}{MAX_JSONB_SAFE_MAGNITUDE:e}"));
}
}
}
}
_ => {
// Structural bytes and whitespace; non-ASCII only occurs
// inside strings, which the arm above copies as slices.
let start = i;
while i < bytes.len() && !matches!(bytes[i], b'"' | b'-' | b'0'..=b'9') {
i += 1;
}
out.push_str(&text[start..i]);
}
}
}
out
}
/// The size Postgres will report for this value as a `jsonb` column —
/// `pg_column_size(value::jsonb)` — computed without touching Postgres.
pub fn jsonb_column_size(value: &Value) -> usize {
// 4-byte varlena header, then the root container. Scalars at the root
// are wrapped in a one-element pseudo-array container.
let mut offset = 4usize;
offset = match value {
Value::Object(_) | Value::Array(_) => container_size(value, offset),
scalar => scalar_container_size(scalar, offset),
};
offset
}
/// A root-level scalar is stored as a one-element array container with the
/// scalar flag; the layout matches a one-element array.
fn scalar_container_size(scalar: &Value, offset: usize) -> usize {
// Container header + one JEntry.
let mut offset = offset + 4 + 4;
offset += element_size(scalar, offset);
offset
}
/// Size contribution of a container (object or array) starting at
/// `offset` bytes from the start of the varlena datum. Returns the offset
/// just past the container. `offset` matters because numerics and nested
/// containers are padded to 4-byte alignment relative to the datum start.
fn container_size(value: &Value, offset: usize) -> usize {
match value {
Value::Object(map) => {
// Header + one JEntry per key and per value, then all key
// bytes, then all values. Keys are processed in PG's storage
// order (length, then bytewise) because alignment padding for
// numeric/container values depends on the running offset.
let mut offset = offset + 4 + 8 * map.len();
let mut keys: Vec<&String> = map.keys().collect();
keys.sort_by(|a, b| {
a.len()
.cmp(&b.len())
.then_with(|| a.as_bytes().cmp(b.as_bytes()))
});
for key in &keys {
offset += key.len();
}
for key in &keys {
offset += element_size(&map[key.as_str()], offset);
}
offset
}
Value::Array(items) => {
let mut offset = offset + 4 + 4 * items.len();
for item in items {
offset += element_size(item, offset);
}
offset
}
_ => unreachable!("container_size only called for objects and arrays"),
}
}
/// Data-section size of one element whose data begins at `offset` from
/// the datum start (its JEntry is accounted for by the parent container).
fn element_size(value: &Value, offset: usize) -> usize {
match value {
Value::Null | Value::Bool(_) => 0,
Value::String(s) => s.len(),
Value::Number(n) => {
// Numerics are stored as a full numeric datum, padded to
// 4-byte alignment from the datum start.
let padding = (4 - offset % 4) % 4;
padding + numeric_datum_size(&n.to_string())
}
Value::Object(_) | Value::Array(_) => {
let padding = (4 - offset % 4) % 4;
padding + (container_size(value, offset + padding) - (offset + padding))
}
}
}
/// The size of the numeric datum PG builds from this JSON number literal:
/// 4-byte varlena header, a 2-byte short (or 4-byte long) numeric header,
/// and two bytes per base-10000 digit group, mirroring `numeric_in`.
fn numeric_datum_size(literal: &str) -> usize {
let (weight, groups, dscale) = parse_numeric(literal);
let header = if short_header_fits(weight, dscale) {
2
} else {
4
};
4 + header + 2 * groups
}
/// PG's short numeric header packs the sign, a 6-bit display scale, and a
/// 7-bit signed weight; it fits when dscale <= 63 and weight is in
/// [-64, 63] base-10000 units (NUMERIC_SHORT_WEIGHT_MIN/MAX in numeric.c).
fn short_header_fits(weight: i32, dscale: u32) -> bool {
dscale <= 0x3F && (-64..=63).contains(&weight)
}
/// Parse a JSON number literal the way `numeric_in` does, returning the
/// base-10000 weight, the count of stored base-10000 digit groups (with
/// zero groups stripped from both ends, as PG does), and the display
/// scale (count of decimal fraction digits after expanding any exponent).
fn parse_numeric(literal: &str) -> (i32, usize, u32) {
let s = literal.strip_prefix('-').unwrap_or(literal);
let (mantissa, exponent) = match s.find(['e', 'E']) {
Some(pos) => (&s[..pos], s[pos + 1..].parse::<i64>().unwrap_or(0)),
None => (s, 0),
};
let (int_part, frac_part) = match mantissa.find('.') {
Some(pos) => (&mantissa[..pos], &mantissa[pos + 1..]),
None => (mantissa, ""),
};
// Expand the exponent into a plain decimal digit string with a point
// position. `digits` is the full significand; `point` is how many of
// those digits sit before the decimal point (may be <= 0).
let mut digits: Vec<u8> = int_part.bytes().chain(frac_part.bytes()).collect();
let mut point = int_part.len() as i64 + exponent;
// Strip leading zeros (they only shift the logical point position
// bookkeeping below, which works on the trimmed significand).
let leading = digits.iter().take_while(|&&b| b == b'0').count();
digits.drain(..leading);
point -= leading as i64;
// dscale is the number of decimal fraction digits PG displays: digits
// after the point, floored at zero, including trailing zeros as
// written.
let dscale = (digits.len() as i64 - point).max(0) as u32;
// Trailing zeros do not become stored digits (PG strips zero groups),
// but they already counted toward dscale above.
while digits.last() == Some(&b'0') {
digits.pop();
}
if digits.is_empty() {
// Zero: no digit groups; weight 0.
return (0, 0, dscale);
}
// Group into base-10000 digits aligned to the decimal point: the
// first group holds ((point - 1) mod 4) + 1 leading decimal digits.
let first_group_len = (((point - 1).rem_euclid(4)) + 1) as usize;
let weight = ((point - first_group_len as i64) / 4) as i32;
let mut groups = 0usize;
let mut index = 0usize;
let mut group_len = first_group_len.min(digits.len());
let mut leading_zero_groups = 0usize;
let mut seen_nonzero = false;
while index < digits.len() {
let group = &digits[index..(index + group_len).min(digits.len())];
let is_zero = group.iter().all(|&b| b == b'0');
if is_zero && !seen_nonzero {
leading_zero_groups += 1;
} else {
seen_nonzero = true;
}
groups += 1;
index += group_len;
group_len = 4;
}
// Strip leading zero groups (weight shrinks with them) and trailing
// zero groups (partially-filled last groups pad with zeros, and fully
// zero tails are not stored).
let weight = weight - leading_zero_groups as i32;
groups -= leading_zero_groups;
// Trailing zero groups: recompute by walking backwards over the
// grouped digits. A trailing group is zero only if every digit in it
// is zero; digits had trailing zeros stripped already, so only a
// group that lies entirely past the last stored digit is zero — which
// cannot happen after the strip. Nothing further to remove.
(weight, groups, dscale)
}
/// Trim person properties under `target_bytes` (measured as
/// `jsonb_column_size`, the constraint's own metric) by removing
/// non-protected properties in alphabetical order, matching the Node.js
/// `trimPropertiesToFitSize` algorithm. Returns `None` when the value
/// already fits (no trimming needed) — and also when trimming every
/// non-protected property still cannot fit, which callers distinguish by
/// re-checking the size; see `TrimResult` for the ergonomic wrapper.
pub enum TrimResult {
/// Already within the target; nothing to do.
Fits,
/// Trimming non-protected properties got it under the target.
Trimmed(Value),
/// Protected properties alone exceed the target; nothing to trim.
CannotFit,
}
pub fn trim_properties_to_fit_size(properties: &Value, target_bytes: usize) -> TrimResult {
let candidates: Vec<String> = match properties.as_object() {
Some(map) => {
let mut keys: Vec<String> = map
.keys()
.filter(|k| can_trim_property(k))
.cloned()
.collect();
keys.sort();
keys
}
None => Vec::new(),
};
trim_properties_with_candidates(properties, target_bytes, candidates)
}
/// Same contract as [`trim_properties_to_fit_size`], but the caller names
/// which keys may be removed and in what order — removal walks `candidates`
/// front to back. The caller is responsible for excluding protected
/// properties from the list; `CannotFit` when the candidates run out before
/// the document fits.
pub fn trim_properties_with_candidates(
properties: &Value,
target_bytes: usize,
candidates: Vec<String>,
) -> TrimResult {
let mut current = jsonb_column_size(properties);
if current <= target_bytes {
return TrimResult::Fits;
}
let Some(map) = properties.as_object() else {
return TrimResult::CannotFit;
};
// Removal runs in batches sized by each pair's estimated contribution
// (two 4-byte JEntries, the key bytes, and the value's content measured
// at offset 0), with one exact re-measure per batch. Alignment padding
// makes per-pair contributions position-dependent, so the estimate can
// be off by a few bytes per numeric — the re-measure keeps the result
// exact and the deficit shrinks geometrically, bounding the total cost
// at O(document) per pass for a handful of passes. This can remove a
// key or two more than the strict minimum in padding-heavy edge cases.
let mut trimmed = map.clone();
let mut candidates = candidates.into_iter();
while current > target_bytes {
let deficit = current - target_bytes;
let mut estimated_freed = 0usize;
let mut removed_any = false;
for key in candidates.by_ref() {
let Some(value) = trimmed.remove(&key) else {
continue;
};
estimated_freed += 8 + key.len() + element_size(&value, 0);
removed_any = true;
if estimated_freed >= deficit {
break;
}
}
if !removed_any {
return TrimResult::CannotFit;
}
// Measure the map without cloning it: wrap, measure, unwrap.
let wrapped = Value::Object(std::mem::take(&mut trimmed));
current = jsonb_column_size(&wrapped);
let Value::Object(unwrapped) = wrapped else {
unreachable!("wrapped as an object above");
};
trimmed = unwrapped;
}
TrimResult::Trimmed(Value::Object(trimmed))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn trim_of_many_small_keys_is_not_quadratic() {
// ~1.3MB document of 20k tiny trimmable keys plus protected state.
// The former per-removal clone-and-remeasure loop needs billions of
// operations here and effectively never finishes in a debug build;
// the batched trim completes in well under a second. Completion IS
// the assertion of linearity; correctness is asserted on the result.
let mut map = serde_json::Map::new();
map.insert("email".to_string(), json!("[email protected]"));
for i in 0..20_000 {
map.insert(format!("k{i:05}"), json!("v".repeat(40)));
}
let value = Value::Object(map);
let TrimResult::Trimmed(trimmed) = trim_properties_to_fit_size(&value, 524_288) else {
panic!("a document of trimmable keys must trim");
};
assert!(jsonb_column_size(&trimmed) <= 524_288);
assert_eq!(trimmed["email"], "[email protected]");
}
#[test]
fn protected_properties_are_not_trimmable() {
assert!(!can_trim_property("email"));
assert!(!can_trim_property("$browser"));
assert!(!can_trim_property("utm_source"));
assert!(!can_trim_property("$fbc"));
assert!(!can_trim_property("$user_id"));
assert!(!can_trim_property("$initial_referrer"));
assert!(can_trim_property("custom_field"));
}
#[test]
fn empty_containers_have_known_jsonb_sizes() {
// varlena header (4) + container header (4).
assert_eq!(jsonb_column_size(&json!({})), 8);
assert_eq!(jsonb_column_size(&json!([])), 8);
}
#[test]
fn trim_removes_custom_properties_alphabetically() {
let target = 1024;
let mut map = serde_json::Map::new();
map.insert("email".to_string(), json!("[email protected]"));
map.insert("$browser".to_string(), json!("Chrome"));
let big_value = "x".repeat(500);
map.insert("aaa_custom".to_string(), json!(big_value.clone()));
map.insert("bbb_custom".to_string(), json!(big_value.clone()));
map.insert("ccc_custom".to_string(), json!(big_value));
match trim_properties_to_fit_size(&Value::Object(map), target) {
TrimResult::Trimmed(trimmed) => {
let trimmed = trimmed.as_object().unwrap();
assert!(trimmed.contains_key("email"));
assert!(trimmed.contains_key("$browser"));
assert!(!trimmed.contains_key("aaa_custom"));
}
_ => panic!("expected a trim"),
}
}
#[test]
fn trim_reports_fits_and_cannot_fit() {
let small = json!({"email": "[email protected]"});
assert!(matches!(
trim_properties_to_fit_size(&small, 1024),
TrimResult::Fits
));
let protected_oversized = json!({"email": "x".repeat(2_000)});
assert!(matches!(
trim_properties_to_fit_size(&protected_oversized, 1024),
TrimResult::CannotFit
));
}
#[test]
fn sanitize_replaces_nul_in_values_keys_and_nested_structures() {
let mut value = serde_json::json!({
"clean": "ok",
"bad\u{0000}key": "still\u{0000}bad",
"nested": {"inner": ["a\u{0000}b", 1, null]},
});
let stats = sanitize_for_jsonb(&mut value);
assert_eq!(stats.nul_strings, 3);
let map = value.as_object().unwrap();
assert_eq!(map["bad\u{FFFD}key"], "still\u{FFFD}bad");
assert_eq!(map["nested"]["inner"][0], "a\u{FFFD}b");
assert_eq!(map["clean"], "ok");
}
#[test]
fn sanitize_clamps_floats_beyond_the_safe_magnitude() {
let mut value = serde_json::json!({
"huge": 1.7976931348623157e308,
"neg_huge": [-1.5e308],
"fine": 1e306,
"int": u64::MAX,
});
let stats = sanitize_for_jsonb(&mut value);
assert_eq!(stats.clamped_numbers, 2);
assert_eq!(value["huge"], 1e307);
assert_eq!(value["neg_huge"][0], -1e307);
assert_eq!(value["fine"], 1e306);
assert_eq!(value["int"], u64::MAX);
}
#[test]
fn sanitize_canonicalizes_integral_floats_to_postgres_integers() {
let mut value = serde_json::json!({
"exp": 1e17,
"neg": -3e3,
"whole": 2.0,
"neg_zero": -0.0,
"two_63": 9.223372036854776e18,
"i64_min": -9.223372036854776e18,
"frac": 0.5,
});
let stats = sanitize_for_jsonb(&mut value);
assert!(
!stats.changed(),
"value-preserving rewrites are not changes"
);
assert_eq!(value["exp"], 100_000_000_000_000_000u64);
assert_eq!(value["neg"], -3000);
assert_eq!(value["whole"], 2);
assert_eq!(value["neg_zero"], 0);
assert_eq!(value["two_63"], 9_223_372_036_854_775_808u64);
assert_eq!(value["i64_min"], i64::MIN);
assert!(value["frac"].is_f64());
}
#[test]
fn sanitize_keeps_integral_floats_that_no_integer_can_hold() {
// 2^64 saturates the u64 cast; 1e30 is integral but far beyond it.
// Both already round-trip through jsonb as themselves (serde_json
// re-parses their expansions as the same f64), so they stay floats.
let mut value = serde_json::json!([1.8446744073709552e19, 1e30]);
sanitize_for_jsonb(&mut value);
assert!(value[0].is_f64());
assert_eq!(value[0], 1.8446744073709552e19);
assert!(value[1].is_f64());
assert_eq!(value[1], 1e30);
}
#[test]
fn sanitize_leaves_clean_values_untouched() {
let mut value = serde_json::json!({"a": "b", "n": [1, 2.5, "x"]});
let before = value.clone();
assert!(!sanitize_for_jsonb(&mut value).changed());
assert_eq!(value, before);
}
/// PG's expanded rendering of JS `Number.MAX_VALUE`: the mantissa
/// digits followed by 292 zeros — serde_json rejects it, JSON.parse
/// rounds it to exactly `f64::MAX`.
fn max_value_expansion() -> String {
format!("17976931348623157{}", "0".repeat(292))
}
#[test]
fn rewrite_rounds_representable_expansions_to_the_double_js_reads() {
let text = format!("{{\"x\": {}}}", max_value_expansion());
assert!(serde_json::from_str::<Value>(&text).is_err());
let parsed: Value = serde_json::from_str(&rewrite_out_of_range_numbers(&text)).unwrap();
assert_eq!(parsed["x"].as_f64().unwrap(), f64::MAX);
}
#[test]
fn rewrite_clamps_beyond_f64_preserving_sign() {
// 2e308 and a Python-bigint-scale integer, positive and negative.
let text = format!(
"{{\"a\": 2{z308}, \"b\": -2{z308}, \"c\": 1{z400}}}",
z308 = "0".repeat(308),
z400 = "0".repeat(400),
);
assert!(serde_json::from_str::<Value>(&text).is_err());
let parsed: Value = serde_json::from_str(&rewrite_out_of_range_numbers(&text)).unwrap();
assert_eq!(parsed["a"].as_f64().unwrap(), MAX_JSONB_SAFE_MAGNITUDE);
assert_eq!(parsed["b"].as_f64().unwrap(), -MAX_JSONB_SAFE_MAGNITUDE);
assert_eq!(parsed["c"].as_f64().unwrap(), MAX_JSONB_SAFE_MAGNITUDE);
}
#[test]
fn rewrite_never_touches_strings_or_parseable_numbers() {
// Digit runs inside strings — including one shaped exactly like
// an offending token, behind an escaped quote — plus in-range
// numbers, must all pass through byte-identical.
let text = format!(
"{{\"note\": \"big: 2{z}\", \"esc\": \"say \\\"1{z}\\\" loud\", \
\"n\": 1e307, \"tiny\": 0.000123, \"neg\": -42}}",
z = "0".repeat(308),
);
assert_eq!(rewrite_out_of_range_numbers(&text), text);
}
#[test]
fn rewrite_output_reparses_for_every_probe_shape() {
// The PG probe fixtures that motivated the rewriter, as one doc.
let text = format!(
"[{max}, 2{z308}, 1{z400}, 1e-308, 0.5, \"x\"]",
max = max_value_expansion(),
z308 = "0".repeat(308),
z400 = "0".repeat(400),
);
let parsed: Value = serde_json::from_str(&rewrite_out_of_range_numbers(&text)).unwrap();
let items = parsed.as_array().unwrap();
assert_eq!(items[0].as_f64().unwrap(), f64::MAX);
assert_eq!(items[1].as_f64().unwrap(), MAX_JSONB_SAFE_MAGNITUDE);
assert_eq!(items[2].as_f64().unwrap(), MAX_JSONB_SAFE_MAGNITUDE);
assert_eq!(items[3].as_f64().unwrap(), 1e-308);
assert_eq!(items[4].as_f64().unwrap(), 0.5);
assert_eq!(items[5], "x");
}
}