Merge pull request #1389 from vhuynh3000/decrypt_on_demand

add config to decrypt on demand instead of polling
2021-01-05 19:32:22 +01:00 · 2021-01-05 19:32:22 +01:00 · 2c24bf3222
--- a/src/invidious.cr
+++ b/src/invidious.cr
@ -168,7 +168,11 @@ end
 Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config)
 Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config)
 Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY)
-Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new
+
+DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling)
+if config.decrypt_polling
+  Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new(logger)
+end

 if config.statistics_enabled
  Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE)
@ -191,8 +195,6 @@ def popular_videos
  Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
 end

-DECRYPT_FUNCTION = Invidious::Jobs::UpdateDecryptFunctionJob::DECRYPT_FUNCTION
-
 before_all do |env|
  preferences = begin
    Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}")
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@ -67,6 +67,7 @@ class Config
  property channel_threads : Int32                 # Number of threads to use for crawling videos from channels (for updating subscriptions)
  property feed_threads : Int32                    # Number of threads to use for updating feeds
  property db : DBConfig                           # Database configuration
+  property decrypt_polling : Bool = true           # Use polling to keep decryption function up to date
  property full_refresh : Bool                     # Used for crawling channels: threads should check all videos uploaded by a channel
  property https_only : Bool?                      # Used to tell Invidious it is behind a proxy, so links to resources should be https://
  property hmac_key : String?                      # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@ -1,53 +1,73 @@
 alias SigProc = Proc(Array(String), Int32, Array(String))

-def fetch_decrypt_function(id = "CvFH_6DNRCY")
-  document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
-  url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
-  player = YT_POOL.client &.get(url).body
+struct DecryptFunction
+  @decrypt_function = [] of {SigProc, Int32}
+  @decrypt_time = Time.monotonic

-  function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
-  function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
-  function_body = function_body.split(";")[1..-2]
+  def initialize(@use_polling = true)
+  end

-  var_name = function_body[0][0, 2]
-  var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
+  def update_decrypt_function
+    @decrypt_function = fetch_decrypt_function
+  end

-  operations = {} of String => SigProc
-  var_body.split("},").each do |operation|
-    op_name = operation.match(/^[^:]+/).not_nil![0]
-    op_body = operation.match(/\{[^}]+/).not_nil![0]
+  private def fetch_decrypt_function(id = "CvFH_6DNRCY")
+    document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
+    url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
+    player = YT_POOL.client &.get(url).body

-    case op_body
-    when "{a.reverse()"
-      operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
-    when "{a.splice(0,b)"
-      operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
-    else
-      operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
+    function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
+    function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
+    function_body = function_body.split(";")[1..-2]
+
+    var_name = function_body[0][0, 2]
+    var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
+
+    operations = {} of String => SigProc
+    var_body.split("},").each do |operation|
+      op_name = operation.match(/^[^:]+/).not_nil![0]
+      op_body = operation.match(/\{[^}]+/).not_nil![0]
+
+      case op_body
+      when "{a.reverse()"
+        operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
+      when "{a.splice(0,b)"
+        operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
+      else
+        operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
+      end
    end
+
+    decrypt_function = [] of {SigProc, Int32}
+    function_body.each do |function|
+      function = function.lchop(var_name).delete("[].")
+
+      op_name = function.match(/[^\(]+/).not_nil![0]
+      value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
+
+      decrypt_function << {operations[op_name], value}
+    end
+
+    return decrypt_function
  end

-  decrypt_function = [] of {SigProc, Int32}
-  function_body.each do |function|
-    function = function.lchop(var_name).delete("[].")
+  def decrypt_signature(fmt : Hash(String, JSON::Any))
+    return "" if !fmt["s"]? || !fmt["sp"]?

-    op_name = function.match(/[^\(]+/).not_nil![0]
-    value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
+    sp = fmt["sp"].as_s
+    sig = fmt["s"].as_s.split("")
+    if !@use_polling
+      now = Time.monotonic
+      if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0
+        @decrypt_function = fetch_decrypt_function
+        @decrypt_time = Time.monotonic
+      end
+    end

-    decrypt_function << {operations[op_name], value}
+    @decrypt_function.each do |proc, value|
+      sig = proc.call(sig, value)
+    end
+
+    return "&#{sp}=#{sig.join("")}"
  end
-
-  return decrypt_function
-end
-
-def decrypt_signature(fmt : Hash(String, JSON::Any))
-  return "" if !fmt["s"]? || !fmt["sp"]?
-
-  sp = fmt["sp"].as_s
-  sig = fmt["s"].as_s.split("")
-  DECRYPT_FUNCTION.each do |proc, value|
-    sig = proc.call(sig, value)
-  end
-
-  return "&#{sp}=#{sig.join("")}"
 end
--- a/src/invidious/jobs/update_decrypt_function_job.cr
+++ b/src/invidious/jobs/update_decrypt_function_job.cr
@ -1,15 +1,15 @@
 class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
-  DECRYPT_FUNCTION = [] of {SigProc, Int32}
+  private getter logger : Invidious::LogHandler
+
+  def initialize(@logger)
+  end

  def begin
    loop do
      begin
-        decrypt_function = fetch_decrypt_function
-        DECRYPT_FUNCTION.clear
-        decrypt_function.each { |df| DECRYPT_FUNCTION << df }
+        DECRYPT_FUNCTION.update_decrypt_function
      rescue ex
-        # TODO: Log error
-        next
+        logger.error("UpdateDecryptFunctionJob : #{ex.message}")
      ensure
        sleep 1.minute
        Fiber.yield
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@ -580,7 +580,7 @@ struct Video
        s.each do |k, v|
          fmt[k] = JSON::Any.new(v)
        end
-        fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
+        fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
      end

      fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
@ -599,7 +599,7 @@ struct Video
        s.each do |k, v|
          fmt[k] = JSON::Any.new(v)
        end
-        fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
+        fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
      end

      fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")