Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(clickhouse): remove invalid surrogate pairs from properties

Carl-Gerhard Lindesvärd1883ec2170f8
diff --git a/packages/common/src/object.ts b/packages/common/src/object.ts
index 405156cd4..374342b8d 100644
--- a/packages/common/src/object.ts
+++ b/packages/common/src/object.ts
@@ -5,6 +5,13 @@ export function toDots(
obj: Record<string, unknown>,
path = '',
): Record<string, string> {
+ // Clickhouse breaks on insert if a property contains invalid surrogate pairs
+ function removeInvalidSurrogates(value: string): string {
+ const validSurrogatePairRegex =
+ /[\uD800-\uDBFF][\uDC00-\uDFFF]|[^\uD800-\uDFFF]/g;
+ return value.match(validSurrogatePairRegex)?.join('') || '';
+ }
+
return Object.entries(obj).reduce((acc, [key, value]) => {
if (typeof value === 'object' && value !== null) {
return {
@@ -17,10 +24,14 @@ export function toDots(
return acc;
}
+ const cleanedValue =
+ typeof value === 'string'
+ ? removeInvalidSurrogates(value).trim()
+ : String(value);
+
return {
...acc,
- [`${path}${key}`]:
- typeof value === 'string' ? value.trim() : String(value),
+ [`${path}${key}`]: cleanedValue,
};
}, {});
}