Skip to content

Invidious is an alternative front-end to YouTube

fix: fix rendered links and timestamps in video descriptions (#5878)

Fijxub4b6e992fc8d
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
index 18b4122e..2c983269 100644
--- a/src/invidious/videos/description.cr
+++ b/src/invidious/videos/description.cr
@@ -1,6 +1,17 @@
require "json"
require "uri"
+# size < bytesize, so we need to count the number of characters that are
+# two UInt16 wide.
+# Taken from: https://github.com/crystal-lang/crystal/blob/8fa7f90c091aa3757821c04ee243c7ab5f67ac20/src/string/utf16.cr#L18-L20
+private def utf16_length(content : String) : Int32
+ u16_size = 0
+ content.each_char do |char|
+ u16_size += char.ord < 0x1_0000 ? 1 : 2
+ end
+ u16_size
+end
+
private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int
copied = 0
while copied < count
@@ -21,6 +32,8 @@ private def copy_string(str : String::Builder, iter : Iterator, count : Int) : I
str << cp.chr
end
+ # A codepoint from the SMP counts twice
+ copied += 1 if cp > 0xFFFF
copied += 1
end
@@ -38,10 +51,15 @@ def parse_description(desc, video_id : String) : String?
# Slightly faster than HTML.escape, as we're only doing one pass on
# the string instead of five for the standard library
return String.build do |str|
- copy_string(str, content.each_codepoint, content.size)
+ content_size = content.ascii_only? ? content.size : utf16_length(content)
+ copy_string(str, content.each_codepoint, content_size)
end
end
+ # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
+ # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
+ # automatically decoded by the JSON parser. It means that we need to count
+ # copied byte in a special manner, preventing the use of regular string copy.
iter = content.each_codepoint
index = 0
@@ -70,7 +88,8 @@ def parse_description(desc, video_id : String) : String?
end
# Copy the end of the string (past the last command).
- remaining_length = content.size - index
+ content_size = content.ascii_only? ? content.size : utf16_length(content)
+ remaining_length = content_size - index
copy_string(str, iter, remaining_length) if remaining_length > 0
end
end