Invidious is an alternative front-end to YouTube
Remove signature helper completely from Invidious (#5550)
Fijxu07f3894a71f5
diff --git a/config/config.example.yml b/config/config.example.ymlindex 2b99345b..eedd9539 100644--- a/config/config.example.yml+++ b/config/config.example.yml@@ -40,20 +40,6 @@ db:###check_tables: false--##-## Path to an external signature resolver, used to emulate-## the Youtube client's Javascript. If no such server is-## available, some videos will not be playable.-##-## When this setting is commented out, no external-## resolver will be used.-##-## Accepted values: a path to a UNIX socket or "<IP>:<Port>"-## Default: <none>-##-#signature_server:-#### Invidious companion is an external program## for loading the video streams from YouTube servers.@@ -259,19 +245,6 @@ https_only: false### use_innertube_for_captions: false-##-## Send Google session informations. This is useful when Invidious is blocked-## by the message "This helps protect our community."-## See https://github.com/iv-org/invidious/issues/4734.-##-## Warning: These strings gives much more identifiable information to Google!-##-## Accepted values: String-## Default: <none>-##-# po_token: ""-# visitor_data: ""-# -----------------------------# Logging# -----------------------------diff --git a/src/invidious.cr b/src/invidious.crindex 197b150c..7fa0725e 100644--- a/src/invidious.cr+++ b/src/invidious.cr@@ -170,15 +170,6 @@ Invidious::Database.check_integrity(CONFIG){% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}{% end %}-# Misc--DECRYPT_FUNCTION =- if sig_helper_address = CONFIG.signature_server.presence- IV::DecryptFunction.new(sig_helper_address)- else- nil- end-# Start jobsif CONFIG.channel_threads > 0diff --git a/src/invidious/config.cr b/src/invidious/config.crindex 92c510d0..7853d9a3 100644--- a/src/invidious/config.cr+++ b/src/invidious/config.cr@@ -153,9 +153,6 @@ class Config@[YAML::Field(converter: Preferences::FamilyConverter)]property force_resolve : Socket::Family = Socket::Family::UNSPEC- # External signature solver server socket (either a path to a UNIX domain socket or "<IP>:<Port>")- property signature_server : String? = nil-# Port to listen for connections (overridden by command line argument)property port : Int32 = 3000# Host to bind (overridden by command line argument)@@ -170,11 +167,6 @@ class Config# Use Innertube's transcripts API instead of timedtext for closed captionsproperty use_innertube_for_captions : Bool = false- # visitor data ID for Google session- property visitor_data : String? = nil- # poToken for passing bot attestation- property po_token : String? = nil-# Invidious companionproperty invidious_companion : Array(CompanionConfig) = [] of CompanionConfig@@ -262,11 +254,7 @@ class Config{% end %}if config.invidious_companion.present?- # invidious_companion and signature_server can't work together- if config.signature_server- puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."- exit(1)- elsif config.invidious_companion_key.empty?+ if config.invidious_companion_key.empty?puts "Config: Please configure a key if you are using invidious companion."exit(1)elsif config.invidious_companion_key == "CHANGE_ME!!"@@ -284,8 +272,6 @@ class Configcompanion.builtin_proxy = trueendend- elsif config.signature_server- puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/installation/")elseputs("WARNING: Invidious companion is required to view and playback videos. For more information see https://docs.invidious.io/installation/")enddiff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.crdeleted file mode 100644index 6d198a42..00000000--- a/src/invidious/helpers/sig_helper.cr+++ /dev/null@@ -1,349 +0,0 @@-require "uri"-require "socket"-require "socket/tcp_socket"-require "socket/unix_socket"--{% if flag?(:advanced_debug) %}- require "io/hexdump"-{% end %}--private alias NetworkEndian = IO::ByteFormat::NetworkEndian--module Invidious::SigHelper- enum UpdateStatus- Updated- UpdateNotRequired- Error- end-- # -------------------- # Payload types- # --------------------- abstract struct Payload- end-- struct StringPayload < Payload- getter string : String-- def initialize(str : String)- raise Exception.new("SigHelper: String can't be empty") if str.empty?- @string = str- end-- def self.from_bytes(slice : Bytes)- size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice)- if size == 0 # Error code- raise Exception.new("SigHelper: Server encountered an error")- end-- if (slice.bytesize - 2) != size- raise Exception.new("SigHelper: String size mismatch")- end-- if str = String.new(slice[2..])- return self.new(str)- else- raise Exception.new("SigHelper: Can't read string from socket")- end- end-- def to_io(io)- # `.to_u16` raises if there is an overflow during the conversion- io.write_bytes(@string.bytesize.to_u16, NetworkEndian)- io.write(@string.to_slice)- end- end-- private enum Opcode- FORCE_UPDATE = 0- DECRYPT_N_SIGNATURE = 1- DECRYPT_SIGNATURE = 2- GET_SIGNATURE_TIMESTAMP = 3- GET_PLAYER_STATUS = 4- PLAYER_UPDATE_TIMESTAMP = 5- end-- private record Request,- opcode : Opcode,- payload : Payload?-- # ----------------------- # High-level functions- # ------------------------ class Client- @mux : Multiplexor-- def initialize(uri_or_path)- @mux = Multiplexor.new(uri_or_path)- end-- # Forces the server to re-fetch the YouTube player, and extract the necessary- # components from it (nsig function code, sig function code, signature timestamp).- def force_update : UpdateStatus- request = Request.new(Opcode::FORCE_UPDATE, nil)-- value = send_request(request) do |bytes|- IO::ByteFormat::NetworkEndian.decode(UInt16, bytes)- end-- case value- when 0x0000 then return UpdateStatus::Error- when 0xFFFF then return UpdateStatus::UpdateNotRequired- when 0xF44F then return UpdateStatus::Updated- else- code = value.nil? ? "nil" : value.to_s(base: 16)- raise Exception.new("SigHelper: Invalid status code received #{code}")- end- end-- # Decrypt a provided n signature using the server's current nsig function- # code, and return the result (or an error).- def decrypt_n_param(n : String) : String?- request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n))-- n_dec = self.send_request(request) do |bytes|- StringPayload.from_bytes(bytes).string- end-- return n_dec- end-- # Decrypt a provided s signature using the server's current sig function- # code, and return the result (or an error).- def decrypt_sig(sig : String) : String?- request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig))-- sig_dec = self.send_request(request) do |bytes|- StringPayload.from_bytes(bytes).string- end-- return sig_dec- end-- # Return the signature timestamp from the server's current player- def get_signature_timestamp : UInt64?- request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil)-- return self.send_request(request) do |bytes|- IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)- end- end-- # Return the current player's version- def get_player : UInt32?- request = Request.new(Opcode::GET_PLAYER_STATUS, nil)-- return self.send_request(request) do |bytes|- has_player = (bytes[0] == 0xFF)- player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4])- has_player ? player_version : nil- end- end-- # Return when the player was last updated- def get_player_timestamp : UInt64?- request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil)-- return self.send_request(request) do |bytes|- IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)- end- end-- private def send_request(request : Request, &)- channel = @mux.send(request)- slice = channel.receive- return yield slice- rescue ex- LOGGER.debug("SigHelper: Error when sending a request")- LOGGER.trace(ex.inspect_with_backtrace)- return nil- end- end-- # ---------------------- # Low level functions- # ----------------------- class Multiplexor- alias TransactionID = UInt32- record Transaction, channel = ::Channel(Bytes).new-- @prng = Random.new- @mutex = Mutex.new- @queue = {} of TransactionID => Transaction-- @conn : Connection- @uri_or_path : String-- def initialize(@uri_or_path)- @conn = Connection.new(uri_or_path)- listen- end-- def listen : Nil- raise "Socket is closed" if @conn.closed?-- LOGGER.debug("SigHelper: Multiplexor listening")-- spawn do- loop do- begin- receive_data- rescue ex- LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")- # We close the socket because for some reason is not closed.- @conn.close- loop do- begin- @conn = Connection.new(@uri_or_path)- LOGGER.info("SigHelper: Reconnected to SigHelper!")- rescue ex- LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")- sleep 500.milliseconds- next- end- break if [email protected]?- end- end- Fiber.yield- end- end- end-- def send(request : Request)- transaction = Transaction.new- transaction_id = @prng.rand(TransactionID)-- # Add transaction to queue- @mutex.synchronize do- # On a 32-bits random integer, this should never happen. Though, just in case, ...- if @queue[transaction_id]?- raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!")- end-- @queue[transaction_id] = transaction- end-- write_packet(transaction_id, request)-- return transaction.channel- end-- def receive_data- transaction_id, slice = read_packet-- @mutex.synchronize do- if transaction = @queue.delete(transaction_id)- # Remove transaction from queue and send data to the channel- transaction.channel.send(slice)- LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel")- else- raise Exception.new("SigHelper: Received transaction was not in queue")- end- end- end-- # Read a single packet from the socket- private def read_packet : {TransactionID, Bytes}- # Header- transaction_id = @conn.read_bytes(UInt32, NetworkEndian)- length = @conn.read_bytes(UInt32, NetworkEndian)-- LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}")-- if length > 67_000- raise Exception.new("SigHelper: Packet longer than expected (#{length})")- end-- # Payload- slice = Bytes.new(length)- @conn.read(slice) if length > 0-- LOGGER.trace("SigHelper: payload = #{slice}")- LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done")-- return transaction_id, slice- end-- # Write a single packet to the socket- private def write_packet(transaction_id : TransactionID, request : Request)- LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}")-- io = IO::Memory.new(1024)- io.write_bytes(request.opcode.to_u8, NetworkEndian)- io.write_bytes(transaction_id, NetworkEndian)-- if payload = request.payload- payload.to_io(io)- end-- @conn.send(io)- @conn.flush-- LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done")- end- end-- class Connection- @socket : UNIXSocket | TCPSocket-- {% if flag?(:advanced_debug) %}- @io : IO::Hexdump- {% end %}-- def initialize(host_or_path : String)- case host_or_path- when .starts_with?('/')- # Make sure that the file exists- if File.exists?(host_or_path)- @socket = UNIXSocket.new(host_or_path)- else- raise Exception.new("SigHelper: '#{host_or_path}' no such file")- end- when .starts_with?("tcp://")- uri = URI.parse(host_or_path)- @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)- else- uri = URI.parse("tcp://#{host_or_path}")- @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)- end- LOGGER.info("SigHelper: Using helper at '#{host_or_path}'")-- {% if flag?(:advanced_debug) %}- @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true)- {% end %}-- @socket.sync = false- @socket.blocking = false- end-- def closed? : Bool- return @socket.closed?- end-- def close : Nil- @socket.close if [email protected]?- end-- def flush(*args, **options)- @socket.flush(*args, **options)- end-- def send(*args, **options)- @socket.send(*args, **options)- end-- # Wrap IO functions, with added debug tooling if needed- {% for function in %w(read read_bytes write write_bytes) %}- def {{function.id}}(*args, **options)- {% if flag?(:advanced_debug) %}- @io.{{function.id}}(*args, **options)- {% else %}- @socket.{{function.id}}(*args, **options)- {% end %}- end- {% end %}- end-enddiff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.crdeleted file mode 100644index 82a28fc0..00000000--- a/src/invidious/helpers/signatures.cr+++ /dev/null@@ -1,53 +0,0 @@-require "http/params"-require "./sig_helper"--class Invidious::DecryptFunction- @last_update : Time = Time.utc - 42.days-- def initialize(uri_or_path)- @client = SigHelper::Client.new(uri_or_path)- self.check_update- end-- def check_update- # If we have updated in the last 5 minutes, do nothing- return if (Time.utc - @last_update) < 5.minutes-- # Get the amount of time elapsed since when the player was updated, in the- # event where multiple invidious processes are run in parallel.- update_time_elapsed = (@client.get_player_timestamp || 301).seconds-- if update_time_elapsed > 5.minutes- LOGGER.debug("Signature: Player might be outdated, updating")- @client.force_update- @last_update = Time.utc- end- end-- def decrypt_nsig(n : String) : String?- self.check_update- return @client.decrypt_n_param(n)- rescue ex- LOGGER.debug(ex.message || "Signature: Unknown error")- LOGGER.trace(ex.inspect_with_backtrace)- return nil- end-- def decrypt_signature(str : String) : String?- self.check_update- return @client.decrypt_sig(str)- rescue ex- LOGGER.debug(ex.message || "Signature: Unknown error")- LOGGER.trace(ex.inspect_with_backtrace)- return nil- end-- def get_sts : UInt64?- self.check_update- return @client.get_signature_timestamp- rescue ex- LOGGER.debug(ex.message || "Signature: Unknown error")- LOGGER.trace(ex.inspect_with_backtrace)- return nil- end-enddiff --git a/src/invidious/videos.cr b/src/invidious/videos.crindex 348a0a66..0446922f 100644--- a/src/invidious/videos.cr+++ b/src/invidious/videos.cr@@ -326,6 +326,14 @@ enddef fetch_video(id, region)info = extract_video_info(video_id: id)+ if info.nil?+ raise InfoException.new("Invidious companion is not available. \+ Video playback cannot continue. \+ If you are the administrator of this instance, install Invidious companion \+ following the installation instructions \+ <a href=\"https://docs.invidious.io/installation/\">https://docs.invidious.io/installation/</a>")+ end+if reason = info["reason"]?if reason == "Video unavailable"raise NotFoundException.new(reason.as_s || "")diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.crindex 6038dfcf..8114ad68 100644--- a/src/invidious/videos/parser.cr+++ b/src/invidious/videos/parser.cr@@ -53,11 +53,12 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?enddef extract_video_info(video_id : String)- # Init client config for the API- client_config = YoutubeAPI::ClientConfig.new-# Fetch data from the player endpoint- player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)+ player_response = YoutubeAPI.player(video_id: video_id)++ if player_response.nil?+ return nil+ endplayability_status = player_response.dig?("playabilityStatus", "status").try &.as_s@@ -105,37 +106,6 @@ def extract_video_info(video_id : String)params = parse_video_info(video_id, player_response)params["reason"] = JSON::Any.new(reason) if reason- if !CONFIG.invidious_companion.present?- if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?- LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")- players_fallback = {YoutubeAPI::ClientType::TvSimply, YoutubeAPI::ClientType::WebMobile}-- players_fallback.each do |player_fallback|- client_config.client_type = player_fallback-- next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))-- adaptive_formats = player_fallback_response.dig?("streamingData", "adaptiveFormats")- if adaptive_formats && (adaptive_formats.dig?(0, "url") || adaptive_formats.dig?(0, "signatureCipher"))- streaming_data = player_response["streamingData"].as_h- streaming_data["adaptiveFormats"] = adaptive_formats- player_response["streamingData"] = JSON::Any.new(streaming_data)- break- end- rescue InfoException- next LOGGER.warn("Failed to fetch streams with #{player_fallback}")- end- end-- # Seems like video page can still render even without playable streams.- # its better than nothing.- #- # # Were we able to find playable video streams?- # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?- # # No :(- # end- end-{"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|params[f] = player_response[f] if player_response[f]?end@@ -163,7 +133,7 @@ enddef try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")- response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)+ response = YoutubeAPI.player(video_id: id)playability_status = response["playabilityStatus"]["status"]LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")@@ -475,26 +445,15 @@ endprivate def convert_url(fmt)if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }- sp = cfr["sp"]url = URI.parse(cfr["url"])params = url.query_paramsLOGGER.debug("convert_url: Decoding '#{cfr}'")-- unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])- params[sp] = unsig if unsigelseurl = URI.parse(fmt["url"].as_s)params = url.query_paramsend- n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])- params["n"] = n if n-- if token = CONFIG.po_token- params["pot"] = token- end-url.query_params = paramsLOGGER.trace("convert_url: new url is '#{url}'")diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.crindex 6fa8ae0e..dd709920 100644--- a/src/invidious/yt_backend/youtube_api.cr+++ b/src/invidious/yt_backend/youtube_api.cr@@ -199,10 +199,6 @@ module YoutubeAPI# conf_1 = ClientConfig.new(region: "NO")# YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1)#- # # Use the Android client to request video streams URLs- # conf_2 = ClientConfig.new(client_type: ClientType::Android)- # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2)- ##struct ClientConfig# Type of client to emulate.@@ -335,10 +331,6 @@ module YoutubeAPIclient_context["client"]["platform"] = platformend- if CONFIG.visitor_data.is_a?(String)- client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)- end-return client_contextend@@ -455,61 +447,23 @@ module YoutubeAPIend####################################################################- # player(video_id, params, client_config?)+ # player(video_id)#- # Requests the youtubei/v1/player endpoint with the required headers- # and POST data in order to get a JSON reply.+ # Requests the youtubei/v1/player Invidious Companion endpoint with+ # the requested video ID.#- # The requested data is a video ID (`v=` parameter), with some- # additional parameters, formatted as a base64 string.+ # The requested data is a video ID (`v=` parameter).#- # An optional ClientConfig parameter can be passed, too (see- # `struct ClientConfig` above for more details).- #- def player(- video_id : String,- *, # Force the following parameters to be passed by name- params : String,- client_config : ClientConfig | Nil = nil,- )- # Playback context, separate because it can be different between clients- playback_ctx = {- "html5Preference" => "HTML5_PREF_WANTS",- "referer" => "https://www.youtube.com/watch?v=#{video_id}",- } of String => String | Int64-- if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s }- if sts = DECRYPT_FUNCTION.try &.get_sts- playback_ctx["signatureTimestamp"] = sts.to_i64- end- end-- # JSON Request data, required by the API+ def player(video_id : String)+ # JSON Request data, required by Invidious Companiondata = {- "contentCheckOk" => true,- "videoId" => video_id,- "context" => self.make_context(client_config, video_id),- "racyCheckOk" => true,- "user" => {- "lockedSafetyMode" => false,- },- "playbackContext" => {- "contentPlaybackContext" => playback_ctx,- },- "serviceIntegrityDimensions" => {- "poToken" => CONFIG.po_token,- },+ "videoId" => video_id,}- # Append the additional parameters if those were provided- if params != ""- data["params"] = params- end-if CONFIG.invidious_companion.present?return self._post_invidious_companion("/youtubei/v1/player", data)else- return self._post_json("/youtubei/v1/player", data, client_config)+ return nilendend@@ -635,10 +589,6 @@ module YoutubeAPIheaders["User-Agent"] = user_agentend- if CONFIG.visitor_data.is_a?(String)- headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)- end-# LoggingLOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"")LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")