Skip to content

Invidious is an alternative front-end to YouTube

Remove signature helper completely from Invidious (#5550)

Fijxu07f3894a71f5
diff --git a/config/config.example.yml b/config/config.example.yml
index 2b99345b..eedd9539 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -40,20 +40,6 @@ db:
##
#check_tables: false
-
-##
-## Path to an external signature resolver, used to emulate
-## the Youtube client's Javascript. If no such server is
-## available, some videos will not be playable.
-##
-## When this setting is commented out, no external
-## resolver will be used.
-##
-## Accepted values: a path to a UNIX socket or "<IP>:<Port>"
-## Default: <none>
-##
-#signature_server:
-
##
## Invidious companion is an external program
## for loading the video streams from YouTube servers.
@@ -259,19 +245,6 @@ https_only: false
##
# use_innertube_for_captions: false
-##
-## Send Google session informations. This is useful when Invidious is blocked
-## by the message "This helps protect our community."
-## See https://github.com/iv-org/invidious/issues/4734.
-##
-## Warning: These strings gives much more identifiable information to Google!
-##
-## Accepted values: String
-## Default: <none>
-##
-# po_token: ""
-# visitor_data: ""
-
# -----------------------------
# Logging
# -----------------------------
diff --git a/src/invidious.cr b/src/invidious.cr
index 197b150c..7fa0725e 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -170,15 +170,6 @@ Invidious::Database.check_integrity(CONFIG)
{% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}
{% end %}
-# Misc
-
-DECRYPT_FUNCTION =
- if sig_helper_address = CONFIG.signature_server.presence
- IV::DecryptFunction.new(sig_helper_address)
- else
- nil
- end
-
# Start jobs
if CONFIG.channel_threads > 0
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 92c510d0..7853d9a3 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -153,9 +153,6 @@ class Config
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC
- # External signature solver server socket (either a path to a UNIX domain socket or "<IP>:<Port>")
- property signature_server : String? = nil
-
# Port to listen for connections (overridden by command line argument)
property port : Int32 = 3000
# Host to bind (overridden by command line argument)
@@ -170,11 +167,6 @@ class Config
# Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false
- # visitor data ID for Google session
- property visitor_data : String? = nil
- # poToken for passing bot attestation
- property po_token : String? = nil
-
# Invidious companion
property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig
@@ -262,11 +254,7 @@ class Config
{% end %}
if config.invidious_companion.present?
- # invidious_companion and signature_server can't work together
- if config.signature_server
- puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
- exit(1)
- elsif config.invidious_companion_key.empty?
+ if config.invidious_companion_key.empty?
puts "Config: Please configure a key if you are using invidious companion."
exit(1)
elsif config.invidious_companion_key == "CHANGE_ME!!"
@@ -284,8 +272,6 @@ class Config
companion.builtin_proxy = true
end
end
- elsif config.signature_server
- puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/installation/")
else
puts("WARNING: Invidious companion is required to view and playback videos. For more information see https://docs.invidious.io/installation/")
end
diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr
deleted file mode 100644
index 6d198a42..00000000
--- a/src/invidious/helpers/sig_helper.cr
+++ /dev/null
@@ -1,349 +0,0 @@
-require "uri"
-require "socket"
-require "socket/tcp_socket"
-require "socket/unix_socket"
-
-{% if flag?(:advanced_debug) %}
- require "io/hexdump"
-{% end %}
-
-private alias NetworkEndian = IO::ByteFormat::NetworkEndian
-
-module Invidious::SigHelper
- enum UpdateStatus
- Updated
- UpdateNotRequired
- Error
- end
-
- # -------------------
- # Payload types
- # -------------------
-
- abstract struct Payload
- end
-
- struct StringPayload < Payload
- getter string : String
-
- def initialize(str : String)
- raise Exception.new("SigHelper: String can't be empty") if str.empty?
- @string = str
- end
-
- def self.from_bytes(slice : Bytes)
- size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice)
- if size == 0 # Error code
- raise Exception.new("SigHelper: Server encountered an error")
- end
-
- if (slice.bytesize - 2) != size
- raise Exception.new("SigHelper: String size mismatch")
- end
-
- if str = String.new(slice[2..])
- return self.new(str)
- else
- raise Exception.new("SigHelper: Can't read string from socket")
- end
- end
-
- def to_io(io)
- # `.to_u16` raises if there is an overflow during the conversion
- io.write_bytes(@string.bytesize.to_u16, NetworkEndian)
- io.write(@string.to_slice)
- end
- end
-
- private enum Opcode
- FORCE_UPDATE = 0
- DECRYPT_N_SIGNATURE = 1
- DECRYPT_SIGNATURE = 2
- GET_SIGNATURE_TIMESTAMP = 3
- GET_PLAYER_STATUS = 4
- PLAYER_UPDATE_TIMESTAMP = 5
- end
-
- private record Request,
- opcode : Opcode,
- payload : Payload?
-
- # ----------------------
- # High-level functions
- # ----------------------
-
- class Client
- @mux : Multiplexor
-
- def initialize(uri_or_path)
- @mux = Multiplexor.new(uri_or_path)
- end
-
- # Forces the server to re-fetch the YouTube player, and extract the necessary
- # components from it (nsig function code, sig function code, signature timestamp).
- def force_update : UpdateStatus
- request = Request.new(Opcode::FORCE_UPDATE, nil)
-
- value = send_request(request) do |bytes|
- IO::ByteFormat::NetworkEndian.decode(UInt16, bytes)
- end
-
- case value
- when 0x0000 then return UpdateStatus::Error
- when 0xFFFF then return UpdateStatus::UpdateNotRequired
- when 0xF44F then return UpdateStatus::Updated
- else
- code = value.nil? ? "nil" : value.to_s(base: 16)
- raise Exception.new("SigHelper: Invalid status code received #{code}")
- end
- end
-
- # Decrypt a provided n signature using the server's current nsig function
- # code, and return the result (or an error).
- def decrypt_n_param(n : String) : String?
- request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n))
-
- n_dec = self.send_request(request) do |bytes|
- StringPayload.from_bytes(bytes).string
- end
-
- return n_dec
- end
-
- # Decrypt a provided s signature using the server's current sig function
- # code, and return the result (or an error).
- def decrypt_sig(sig : String) : String?
- request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig))
-
- sig_dec = self.send_request(request) do |bytes|
- StringPayload.from_bytes(bytes).string
- end
-
- return sig_dec
- end
-
- # Return the signature timestamp from the server's current player
- def get_signature_timestamp : UInt64?
- request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil)
-
- return self.send_request(request) do |bytes|
- IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
- end
- end
-
- # Return the current player's version
- def get_player : UInt32?
- request = Request.new(Opcode::GET_PLAYER_STATUS, nil)
-
- return self.send_request(request) do |bytes|
- has_player = (bytes[0] == 0xFF)
- player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4])
- has_player ? player_version : nil
- end
- end
-
- # Return when the player was last updated
- def get_player_timestamp : UInt64?
- request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil)
-
- return self.send_request(request) do |bytes|
- IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
- end
- end
-
- private def send_request(request : Request, &)
- channel = @mux.send(request)
- slice = channel.receive
- return yield slice
- rescue ex
- LOGGER.debug("SigHelper: Error when sending a request")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
- end
-
- # ---------------------
- # Low level functions
- # ---------------------
-
- class Multiplexor
- alias TransactionID = UInt32
- record Transaction, channel = ::Channel(Bytes).new
-
- @prng = Random.new
- @mutex = Mutex.new
- @queue = {} of TransactionID => Transaction
-
- @conn : Connection
- @uri_or_path : String
-
- def initialize(@uri_or_path)
- @conn = Connection.new(uri_or_path)
- listen
- end
-
- def listen : Nil
- raise "Socket is closed" if @conn.closed?
-
- LOGGER.debug("SigHelper: Multiplexor listening")
-
- spawn do
- loop do
- begin
- receive_data
- rescue ex
- LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")
- # We close the socket because for some reason is not closed.
- @conn.close
- loop do
- begin
- @conn = Connection.new(@uri_or_path)
- LOGGER.info("SigHelper: Reconnected to SigHelper!")
- rescue ex
- LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
- sleep 500.milliseconds
- next
- end
- break if [email protected]?
- end
- end
- Fiber.yield
- end
- end
- end
-
- def send(request : Request)
- transaction = Transaction.new
- transaction_id = @prng.rand(TransactionID)
-
- # Add transaction to queue
- @mutex.synchronize do
- # On a 32-bits random integer, this should never happen. Though, just in case, ...
- if @queue[transaction_id]?
- raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!")
- end
-
- @queue[transaction_id] = transaction
- end
-
- write_packet(transaction_id, request)
-
- return transaction.channel
- end
-
- def receive_data
- transaction_id, slice = read_packet
-
- @mutex.synchronize do
- if transaction = @queue.delete(transaction_id)
- # Remove transaction from queue and send data to the channel
- transaction.channel.send(slice)
- LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel")
- else
- raise Exception.new("SigHelper: Received transaction was not in queue")
- end
- end
- end
-
- # Read a single packet from the socket
- private def read_packet : {TransactionID, Bytes}
- # Header
- transaction_id = @conn.read_bytes(UInt32, NetworkEndian)
- length = @conn.read_bytes(UInt32, NetworkEndian)
-
- LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}")
-
- if length > 67_000
- raise Exception.new("SigHelper: Packet longer than expected (#{length})")
- end
-
- # Payload
- slice = Bytes.new(length)
- @conn.read(slice) if length > 0
-
- LOGGER.trace("SigHelper: payload = #{slice}")
- LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done")
-
- return transaction_id, slice
- end
-
- # Write a single packet to the socket
- private def write_packet(transaction_id : TransactionID, request : Request)
- LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}")
-
- io = IO::Memory.new(1024)
- io.write_bytes(request.opcode.to_u8, NetworkEndian)
- io.write_bytes(transaction_id, NetworkEndian)
-
- if payload = request.payload
- payload.to_io(io)
- end
-
- @conn.send(io)
- @conn.flush
-
- LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done")
- end
- end
-
- class Connection
- @socket : UNIXSocket | TCPSocket
-
- {% if flag?(:advanced_debug) %}
- @io : IO::Hexdump
- {% end %}
-
- def initialize(host_or_path : String)
- case host_or_path
- when .starts_with?('/')
- # Make sure that the file exists
- if File.exists?(host_or_path)
- @socket = UNIXSocket.new(host_or_path)
- else
- raise Exception.new("SigHelper: '#{host_or_path}' no such file")
- end
- when .starts_with?("tcp://")
- uri = URI.parse(host_or_path)
- @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
- else
- uri = URI.parse("tcp://#{host_or_path}")
- @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
- end
- LOGGER.info("SigHelper: Using helper at '#{host_or_path}'")
-
- {% if flag?(:advanced_debug) %}
- @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true)
- {% end %}
-
- @socket.sync = false
- @socket.blocking = false
- end
-
- def closed? : Bool
- return @socket.closed?
- end
-
- def close : Nil
- @socket.close if [email protected]?
- end
-
- def flush(*args, **options)
- @socket.flush(*args, **options)
- end
-
- def send(*args, **options)
- @socket.send(*args, **options)
- end
-
- # Wrap IO functions, with added debug tooling if needed
- {% for function in %w(read read_bytes write write_bytes) %}
- def {{function.id}}(*args, **options)
- {% if flag?(:advanced_debug) %}
- @io.{{function.id}}(*args, **options)
- {% else %}
- @socket.{{function.id}}(*args, **options)
- {% end %}
- end
- {% end %}
- end
-end
diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr
deleted file mode 100644
index 82a28fc0..00000000
--- a/src/invidious/helpers/signatures.cr
+++ /dev/null
@@ -1,53 +0,0 @@
-require "http/params"
-require "./sig_helper"
-
-class Invidious::DecryptFunction
- @last_update : Time = Time.utc - 42.days
-
- def initialize(uri_or_path)
- @client = SigHelper::Client.new(uri_or_path)
- self.check_update
- end
-
- def check_update
- # If we have updated in the last 5 minutes, do nothing
- return if (Time.utc - @last_update) < 5.minutes
-
- # Get the amount of time elapsed since when the player was updated, in the
- # event where multiple invidious processes are run in parallel.
- update_time_elapsed = (@client.get_player_timestamp || 301).seconds
-
- if update_time_elapsed > 5.minutes
- LOGGER.debug("Signature: Player might be outdated, updating")
- @client.force_update
- @last_update = Time.utc
- end
- end
-
- def decrypt_nsig(n : String) : String?
- self.check_update
- return @client.decrypt_n_param(n)
- rescue ex
- LOGGER.debug(ex.message || "Signature: Unknown error")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
-
- def decrypt_signature(str : String) : String?
- self.check_update
- return @client.decrypt_sig(str)
- rescue ex
- LOGGER.debug(ex.message || "Signature: Unknown error")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
-
- def get_sts : UInt64?
- self.check_update
- return @client.get_signature_timestamp
- rescue ex
- LOGGER.debug(ex.message || "Signature: Unknown error")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
-end
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 348a0a66..0446922f 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -326,6 +326,14 @@ end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
+ if info.nil?
+ raise InfoException.new("Invidious companion is not available. \
+ Video playback cannot continue. \
+ If you are the administrator of this instance, install Invidious companion \
+ following the installation instructions \
+ <a href=\"https://docs.invidious.io/installation/\">https://docs.invidious.io/installation/</a>")
+ end
+
if reason = info["reason"]?
if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "")
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index 6038dfcf..8114ad68 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -53,11 +53,12 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
end
def extract_video_info(video_id : String)
- # Init client config for the API
- client_config = YoutubeAPI::ClientConfig.new
-
# Fetch data from the player endpoint
- player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+ player_response = YoutubeAPI.player(video_id: video_id)
+
+ if player_response.nil?
+ return nil
+ end
playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
@@ -105,37 +106,6 @@ def extract_video_info(video_id : String)
params = parse_video_info(video_id, player_response)
params["reason"] = JSON::Any.new(reason) if reason
- if !CONFIG.invidious_companion.present?
- if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
- players_fallback = {YoutubeAPI::ClientType::TvSimply, YoutubeAPI::ClientType::WebMobile}
-
- players_fallback.each do |player_fallback|
- client_config.client_type = player_fallback
-
- next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
-
- adaptive_formats = player_fallback_response.dig?("streamingData", "adaptiveFormats")
- if adaptive_formats && (adaptive_formats.dig?(0, "url") || adaptive_formats.dig?(0, "signatureCipher"))
- streaming_data = player_response["streamingData"].as_h
- streaming_data["adaptiveFormats"] = adaptive_formats
- player_response["streamingData"] = JSON::Any.new(streaming_data)
- break
- end
- rescue InfoException
- next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
- end
- end
-
- # Seems like video page can still render even without playable streams.
- # its better than nothing.
- #
- # # Were we able to find playable video streams?
- # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- # # No :(
- # end
- end
-
{"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
params[f] = player_response[f] if player_response[f]?
end
@@ -163,7 +133,7 @@ end
def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
- response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
+ response = YoutubeAPI.player(video_id: id)
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
@@ -475,26 +445,15 @@ end
private def convert_url(fmt)
if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
- sp = cfr["sp"]
url = URI.parse(cfr["url"])
params = url.query_params
LOGGER.debug("convert_url: Decoding '#{cfr}'")
-
- unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
- params[sp] = unsig if unsig
else
url = URI.parse(fmt["url"].as_s)
params = url.query_params
end
- n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
- params["n"] = n if n
-
- if token = CONFIG.po_token
- params["pot"] = token
- end
-
url.query_params = params
LOGGER.trace("convert_url: new url is '#{url}'")
diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index 6fa8ae0e..dd709920 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -199,10 +199,6 @@ module YoutubeAPI
# conf_1 = ClientConfig.new(region: "NO")
# YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1)
#
- # # Use the Android client to request video streams URLs
- # conf_2 = ClientConfig.new(client_type: ClientType::Android)
- # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2)
- #
#
struct ClientConfig
# Type of client to emulate.
@@ -335,10 +331,6 @@ module YoutubeAPI
client_context["client"]["platform"] = platform
end
- if CONFIG.visitor_data.is_a?(String)
- client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)
- end
-
return client_context
end
@@ -455,61 +447,23 @@ module YoutubeAPI
end
####################################################################
- # player(video_id, params, client_config?)
+ # player(video_id)
#
- # Requests the youtubei/v1/player endpoint with the required headers
- # and POST data in order to get a JSON reply.
+ # Requests the youtubei/v1/player Invidious Companion endpoint with
+ # the requested video ID.
#
- # The requested data is a video ID (`v=` parameter), with some
- # additional parameters, formatted as a base64 string.
+ # The requested data is a video ID (`v=` parameter).
#
- # An optional ClientConfig parameter can be passed, too (see
- # `struct ClientConfig` above for more details).
- #
- def player(
- video_id : String,
- *, # Force the following parameters to be passed by name
- params : String,
- client_config : ClientConfig | Nil = nil,
- )
- # Playback context, separate because it can be different between clients
- playback_ctx = {
- "html5Preference" => "HTML5_PREF_WANTS",
- "referer" => "https://www.youtube.com/watch?v=#{video_id}",
- } of String => String | Int64
-
- if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s }
- if sts = DECRYPT_FUNCTION.try &.get_sts
- playback_ctx["signatureTimestamp"] = sts.to_i64
- end
- end
-
- # JSON Request data, required by the API
+ def player(video_id : String)
+ # JSON Request data, required by Invidious Companion
data = {
- "contentCheckOk" => true,
- "videoId" => video_id,
- "context" => self.make_context(client_config, video_id),
- "racyCheckOk" => true,
- "user" => {
- "lockedSafetyMode" => false,
- },
- "playbackContext" => {
- "contentPlaybackContext" => playback_ctx,
- },
- "serviceIntegrityDimensions" => {
- "poToken" => CONFIG.po_token,
- },
+ "videoId" => video_id,
}
- # Append the additional parameters if those were provided
- if params != ""
- data["params"] = params
- end
-
if CONFIG.invidious_companion.present?
return self._post_invidious_companion("/youtubei/v1/player", data)
else
- return self._post_json("/youtubei/v1/player", data, client_config)
+ return nil
end
end
@@ -635,10 +589,6 @@ module YoutubeAPI
headers["User-Agent"] = user_agent
end
- if CONFIG.visitor_data.is_a?(String)
- headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)
- end
-
# Logging
LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"")
LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")