Invidious is an alternative front-end to YouTube
Encapsulate helpers constants and functions inside it's own `Helpers` module (#5639)
Fijxu73c749f13f15
diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.crindex 3761dde5..34a21426 100644--- a/src/invidious/channels/community.cr+++ b/src/invidious/channels/community.cr@@ -127,7 +127,7 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_singreply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")- json.field "content", html_to_content(content_html)+ json.field "content", Helpers.html_to_content(content_html)json.field "contentHtml", content_htmljson.field "published", published.to_unixdiff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.crindex 8b798798..78a569ab 100644--- a/src/invidious/comments/youtube.cr+++ b/src/invidious/comments/youtube.cr@@ -254,7 +254,7 @@ module Invidious::Commentsendcontent_html = html_content || ""- json.field "content", html_to_content(content_html)+ json.field "content", Helpers.html_to_content(content_html)json.field "contentHtml", content_htmlif published_text != nildiff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.crindex ab694b1f..a08135ea 100644--- a/src/invidious/helpers/helpers.cr+++ b/src/invidious/helpers/helpers.cr@@ -1,7 +1,5 @@require "./macros"-TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}-struct Nonceinclude DB::Serializable@@ -24,60 +22,124 @@ struct Annotationproperty annotations : Stringend-def html_to_content(description_html : String)- description = description_html.gsub(/(<br>)|(<br\/>)/, {- "<br>": "\n",- "<br/>": "\n",- })+module Helpers+ extend self- if !description.empty?- description = XML.parse_html(description).content.strip("\n ")- end+ private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}- return description-end+ def html_to_content(description_html : String)+ description = description_html.gsub(/(<br>)|(<br\/>)/, {+ "<br>": "\n",+ "<br/>": "\n",+ })++ if !description.empty?+ description = XML.parse_html(description).content.strip("\n ")+ end-def cache_annotation(id, annotations)- if !CONFIG.cache_annotations- return+ return descriptionend- body = XML.parse(annotations)- nodeset = body.xpath_nodes(%q(/document/annotations/annotation))+ def cache_annotation(id, annotations)+ if !CONFIG.cache_annotations+ return+ end++ body = XML.parse(annotations)+ nodeset = body.xpath_nodes(%q(/document/annotations/annotation))- return if nodeset == 0+ return if nodeset == 0- has_legacy_annotations = false- nodeset.each do |node|- if !{"branding", "card", "drawer"}.includes? node["type"]?- has_legacy_annotations = true- break+ has_legacy_annotations = false+ nodeset.each do |node|+ if !{"branding", "card", "drawer"}.includes? node["type"]?+ has_legacy_annotations = true+ break+ endend++ Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotationsend- Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations-end+ def create_notification_stream(env, topics, connection_channel)+ connection = Channel(PQ::Notification).new(8)+ connection_channel.send({true, connection})++ locale = env.get("preferences").as(Preferences).locale++ since = env.params.query["since"]?.try &.to_i?+ id = 0++ if topics.includes? "debug"+ spawn do+ begin+ loop do+ time_span = [0, 0, 0, 0]+ time_span[rand(4)] = rand(30) + 5+ published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])+ video_id = TEST_IDS[rand(TEST_IDS.size)]++ video = get_video(video_id)+ video.published = published+ response = JSON.parse(video.to_json(locale, nil))++ env.response.puts "id: #{id}"+ env.response.puts "data: #{response.to_json}"+ env.response.puts+ env.response.flush-def create_notification_stream(env, topics, connection_channel)- connection = Channel(PQ::Notification).new(8)- connection_channel.send({true, connection})+ id += 1- locale = env.get("preferences").as(Preferences).locale+ sleep 1.minute+ Fiber.yield+ end+ rescue ex+ end+ end+ end- since = env.params.query["since"]?.try &.to_i?- id = 0+ spawn do+ begin+ if since+ since_unix = Time.unix(since.not_nil!)++ topics.try &.each do |topic|+ case topic+ when .match(/UC[A-Za-z0-9_-]{22}/)+ Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|+ response = JSON.parse(video.to_json(locale))++ env.response.puts "id: #{id}"+ env.response.puts "data: #{response.to_json}"+ env.response.puts+ env.response.flush++ id += 1+ end+ else+ # TODO+ end+ end+ end+ end+ end- if topics.includes? "debug"spawn dobeginloop do- time_span = [0, 0, 0, 0]- time_span[rand(4)] = rand(30) + 5- published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])- video_id = TEST_IDS[rand(TEST_IDS.size)]+ event = connection.receive++ notification = JSON.parse(event.payload)+ topic = notification["topic"].as_s+ video_id = notification["videoId"].as_s+ published = notification["published"].as_i64++ if !topics.try &.includes? topic+ next+ endvideo = get_video(video_id)- video.published = published+ video.published = Time.unix(published)response = JSON.parse(video.to_json(locale, nil))env.response.puts "id: #{id}"@@ -86,65 +148,20 @@ def create_notification_stream(env, topics, connection_channel)env.response.flushid += 1-- sleep 1.minute- Fiber.yieldendrescue ex+ ensure+ connection_channel.send({false, connection})endend- end- spawn do- begin- if since- since_unix = Time.unix(since.not_nil!)-- topics.try &.each do |topic|- case topic- when .match(/UC[A-Za-z0-9_-]{22}/)- Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|- response = JSON.parse(video.to_json(locale))-- env.response.puts "id: #{id}"- env.response.puts "data: #{response.to_json}"- env.response.puts- env.response.flush-- id += 1- end- else- # TODO- end- end- end- end- end-- spawn dobegin+ # Send heartbeatloop do- event = connection.receive-- notification = JSON.parse(event.payload)- topic = notification["topic"].as_s- video_id = notification["videoId"].as_s- published = notification["published"].as_i64-- if !topics.try &.includes? topic- next- end-- video = get_video(video_id)- video.published = Time.unix(published)- response = JSON.parse(video.to_json(locale, nil))-- env.response.puts "id: #{id}"- env.response.puts "data: #{response.to_json}"+ env.response.puts ":keepalive #{Time.utc.to_unix}"env.response.putsenv.response.flush-- id += 1+ sleep (20 + rand(11)).secondsendrescue exensure@@ -152,51 +169,38 @@ def create_notification_stream(env, topics, connection_channel)endend- begin- # Send heartbeat- loop do- env.response.puts ":keepalive #{Time.utc.to_unix}"- env.response.puts- env.response.flush- sleep (20 + rand(11)).seconds- end- rescue ex- ensure- connection_channel.send({false, connection})+ def extract_initial_data(body) : Hash(String, JSON::Any)+ return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>{.*?});<\/script>/mx).try &.["info"] || "{}").as_hend-end--def extract_initial_data(body) : Hash(String, JSON::Any)- return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>{.*?});<\/script>/mx).try &.["info"] || "{}").as_h-end-def proxy_file(response, env)- if response.headers.includes_word?("Content-Encoding", "gzip")- Compress::Gzip::Writer.open(env.response) do |deflate|- IO.copy response.body_io, deflate- end- elsif response.headers.includes_word?("Content-Encoding", "deflate")- Compress::Deflate::Writer.open(env.response) do |deflate|- IO.copy response.body_io, deflate+ def proxy_file(response, env)+ if response.headers.includes_word?("Content-Encoding", "gzip")+ Compress::Gzip::Writer.open(env.response) do |deflate|+ IO.copy response.body_io, deflate+ end+ elsif response.headers.includes_word?("Content-Encoding", "deflate")+ Compress::Deflate::Writer.open(env.response) do |deflate|+ IO.copy response.body_io, deflate+ end+ else+ IO.copy response.body_io, env.responseend- else- IO.copy response.body_io, env.responseend-end-# Fetch the playback requests tracker from the statistics endpoint.-#-# Creates a new tracker when unavailable.-def get_playback_statistic- if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?- tracker = {- "totalRequests" => 0_i64,- "successfulRequests" => 0_i64,- "ratio" => 0_f64,- }-- Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker- end+ # Fetch the playback requests tracker from the statistics endpoint.+ #+ # Creates a new tracker when unavailable.+ def get_playback_statistic+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?+ tracker = {+ "totalRequests" => 0_i64,+ "successfulRequests" => 0_i64,+ "ratio" => 0_f64,+ }++ Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker+ end- return tracker.as(Hash(String, Int64 | Float64))+ return tracker.as(Hash(String, Int64 | Float64))+ endenddiff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.crindex df040d06..9d0cc588 100644--- a/src/invidious/helpers/serialized_yt_data.cr+++ b/src/invidious/helpers/serialized_yt_data.cr@@ -53,7 +53,7 @@ struct SearchVideoxml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")end- xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }+ xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Helpers.html_to_content(self.description_html) }endend@@ -63,7 +63,7 @@ struct SearchVideoxml.element("media:title") { xml.text self.title }xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",width: "320", height: "180")- xml.element("media:description") { xml.text html_to_content(self.description_html) }+ xml.element("media:description") { xml.text Helpers.html_to_content(self.description_html) }endxml.element("media:community") do@@ -111,7 +111,7 @@ struct SearchVideoInvidious::JSONify::APIv1.thumbnails(json, self.id)end- json.field "description", html_to_content(self.description_html)+ json.field "description", Helpers.html_to_content(self.description_html)json.field "descriptionHtml", self.description_htmljson.field "viewCount", self.views@@ -255,7 +255,7 @@ struct SearchChanneljson.field "videoCount", self.video_countjson.field "channelHandle", self.channel_handle- json.field "description", html_to_content(self.description_html)+ json.field "description", Helpers.html_to_content(self.description_html)json.field "descriptionHtml", self.description_htmlendenddiff --git a/src/invidious/mixes.cr b/src/invidious/mixes.crindex 28ff0ff6..7dd8a830 100644--- a/src/invidious/mixes.cr+++ b/src/invidious/mixes.cr@@ -27,7 +27,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)- initial_data = extract_initial_data(response.body)+ initial_data = Helpers.extract_initial_data(response.body)if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?raise InfoException.new("Could not create mix.")diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.crindex 5c226803..231d4bbc 100644--- a/src/invidious/playlists.cr+++ b/src/invidious/playlists.cr@@ -199,7 +199,7 @@ struct InvidiousPlaylistjson.field "authorUrl", niljson.field "authorThumbnails", [] of String- json.field "description", html_to_content(self.description_html)+ json.field "description", Helpers.html_to_content(self.description_html)json.field "descriptionHtml", self.description_htmljson.field "videoCount", self.video_countdiff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.crindex a35d2f2b..7021321f 100644--- a/src/invidious/routes/api/v1/authenticated.cr+++ b/src/invidious/routes/api/v1/authenticated.cr@@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Authenticated# topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000)# topics ||= [] of String- # create_notification_stream(env, topics, connection_channel)+ # Helpers.create_notification_stream(env, topics, connection_channel)# enddef self.get_preferences(env)@@ -485,6 +485,6 @@ module Invidious::Routes::API::V1::Authenticatedtopics = raw_topics.try &.split(",").uniq.first(1000)topics ||= [] of String- create_notification_stream(env, topics, CONNECTION_CHANNEL)+ Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL)endenddiff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.crindex f8060342..b0545bdd 100644--- a/src/invidious/routes/api/v1/channels.cr+++ b/src/invidious/routes/api/v1/channels.cr@@ -97,7 +97,7 @@ module Invidious::Routes::API::V1::Channelsjson.field "autoGenerated", channel.auto_generatedjson.field "ageGated", channel.is_age_gatedjson.field "isFamilyFriendly", channel.is_family_friendly- json.field "description", html_to_content(channel.description_html)+ json.field "description", Helpers.html_to_content(channel.description_html)json.field "descriptionHtml", channel.description_htmljson.field "allowedRegions", channel.allowed_regionsdiff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.crindex fc3de695..5f98321b 100644--- a/src/invidious/routes/api/v1/videos.cr+++ b/src/invidious/routes/api/v1/videos.cr@@ -300,7 +300,7 @@ module Invidious::Routes::API::V1::Videosannotations = response.body- cache_annotation(id, annotations)+ Helpers.cache_annotation(id, annotations)endelse # "youtube"response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.crindex 51d85dfe..c06955c0 100644--- a/src/invidious/routes/images.cr+++ b/src/invidious/routes/images.cr@@ -96,7 +96,7 @@ module Invidious::Routes::Imagesbreakend- proxy_file(response, env)+ Helpers.proxy_file(response, env)endrescue exend@@ -148,6 +148,6 @@ module Invidious::Routes::Imagesreturn env.response.headers.delete("Transfer-Encoding")end- return proxy_file(response, env)+ return Helpers.proxy_file(response, env)endenddiff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.crindex 7c01aa36..f16a5a58 100644--- a/src/invidious/routes/video_playback.cr+++ b/src/invidious/routes/video_playback.cr@@ -83,7 +83,7 @@ module Invidious::Routes::VideoPlayback# Remove the Range header added previously.headers.delete("Range") if range_header.nil?- playback_statistics = get_playback_statistic()+ playback_statistics = Helpers.get_playback_statisticplayback_statistics["totalRequests"] += 1if response.status_code >= 400@@ -195,7 +195,7 @@ module Invidious::Routes::VideoPlaybackendend- proxy_file(resp, env)+ Helpers.proxy_file(resp, env)endrescue exif ex.message != "Error reading socket: Connection reset by peer"diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.crindex 25edb936..ecf9aeb3 100644--- a/src/invidious/search/processors.cr+++ b/src/invidious/search/processors.cr@@ -21,7 +21,7 @@ module Invidious::Searchif response.status_code == 404response = YT_POOL.client &.get("/user/#{query.channel}")response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404- initial_data = extract_initial_data(response.body)+ initial_data = Helpers.extract_initial_data(response.body)ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)raise ChannelSearchException.new(query.channel) if !ucidelsediff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.crindex b52503c9..3fa468ba 100644--- a/src/invidious/user/exports.cr+++ b/src/invidious/user/exports.cr@@ -15,7 +15,7 @@ struct Invidious::Userplaylists.each do |playlist|json.object dojson.field "title", playlist.title- json.field "description", html_to_content(playlist.description_html)+ json.field "description", Helpers.html_to_content(playlist.description_html)json.field "privacy", playlist.privacy.to_sjson.field "videos" dojson.array dodiff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.crindex 8114ad68..d9107d5a 100644--- a/src/invidious/videos/parser.cr+++ b/src/invidious/videos/parser.cr@@ -84,7 +84,7 @@ def extract_video_info(video_id : String)# Although technically not a call to /videoplayback the fact that YouTube is returning the# wrong video means that we should count it as a failure.- get_playback_statistic()["totalRequests"] += 1+ Helpers.get_playback_statistic["totalRequests"] += 1return {"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),