Add HTTPClient pool

2019-10-25 12:58:16 -04:00 · 2019-10-25 12:58:16 -04:00 · 6930570fa2
--- a/shard.yml
+++ b/shard.yml
@ -18,6 +18,9 @@ dependencies:
  kemal:
    github: kemalcr/kemal
    version: ~> 0.26.0
+  pool:
+    github: ysbaddaden/pool
+    version: ~> 0.2.3

 crystal: 0.31.1

--- a/src/invidious.cr
+++ b/src/invidious.cr
@ -46,6 +46,7 @@ PUBSUB_URL      = URI.parse("https://pubsubhubbub.appspot.com")
 REDDIT_URL      = URI.parse("https://www.reddit.com")
 TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com")
 YT_URL          = URI.parse("https://www.youtube.com")
+YT_IMG_URL      = URI.parse("https://i.ytimg.com")

 CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@ -91,6 +92,9 @@ LOCALES = {
  "zh-TW" => load_locale("zh-TW"),
 }

+YT_POOL     = HTTPPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+
 config = CONFIG
 logger = Invidious::LogHandler.new

@ -658,8 +662,7 @@ get "/embed/:id" do |env|

    next env.redirect url
  when "live_stream"
-    client = make_client(YT_URL)
-    response = client.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
+    response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
    video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]

    env.params.query.delete_all("channel")
@ -2250,8 +2253,7 @@ get "/modify_notifications" do |env|
    headers = HTTP::Headers.new
    headers["Cookie"] = env.request.headers["Cookie"]

-    client = make_client(YT_URL)
-    html = client.get("/subscription_manager?disable_polymer=1", headers)
+    html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)

    cookies = HTTP::Cookies.from_headers(headers)
    html.cookies.each do |cookie|
@ -2280,7 +2282,7 @@ get "/modify_notifications" do |env|
      channel_id = channel.content.lstrip("/channel/").not_nil!
      channel_req["channel_id"] = channel_id

-      client.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
+      YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
    end
  end

@ -2558,8 +2560,7 @@ post "/data_control" do |env|
          if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
            next match["channel"]
          elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
-            client = make_client(YT_URL)
-            response = client.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
+            response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
            document = XML.parse_html(response.body)
            canonical = document.xpath_node(%q(//link[@rel="canonical"]))

@ -3084,8 +3085,7 @@ get "/feed/channel/:ucid" do |env|
    next error_message
  end

-  client = make_client(YT_URL)
-  rss = client.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
+  rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
  rss = XML.parse_html(rss)

  videos = [] of SearchVideo
@ -3229,8 +3229,7 @@ get "/feed/playlist/:plid" do |env|
    end
  end

-  client = make_client(YT_URL)
-  response = client.get("/feeds/videos.xml?playlist_id=#{plid}")
+  response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
  document = XML.parse(response.body)

  document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
@ -3395,14 +3394,13 @@ end
 {"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route|
  get route do |env|
    locale = LOCALES[env.get("preferences").as(Preferences).locale]?
-    client = make_client(YT_URL)

    # Appears to be a bug in routing, having several routes configured
    # as `/a/:a`, `/b/:a`, `/c/:a` results in 404
    value = env.request.resource.split("/")[2]
    body = ""
    {"channel", "user", "c"}.each do |type|
-      response = client.get("/#{type}/#{value}/live?disable_polymer=1")
+      response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
      if response.status_code == 200
        body = response.body
      end
@ -3433,10 +3431,9 @@ end
 get "/c/:user" do |env|
  locale = LOCALES[env.get("preferences").as(Preferences).locale]?

-  client = make_client(YT_URL)
  user = env.params.url["user"]

-  response = client.get("/c/#{user}")
+  response = YT_POOL.client &.get("/c/#{user}")
  document = XML.parse_html(response.body)

  anchor = document.xpath_node(%q(//a[contains(@class,"branded-page-header-title-link")]))
@ -3676,7 +3673,6 @@ get "/api/v1/storyboards/:id" do |env|
  id = env.params.url["id"]
  region = env.params.query["region"]?

-  client = make_client(YT_URL)
  begin
    video = get_video(id, PG_DB, region: region)
  rescue ex : VideoRedirect
@ -3764,7 +3760,6 @@ get "/api/v1/captions/:id" do |env|
  # In future this should be investigated as an alternative, since it does not require
  # getting video info.

-  client = make_client(YT_URL)
  begin
    video = get_video(id, PG_DB, region: region)
  rescue ex : VideoRedirect
@ -3823,7 +3818,7 @@ get "/api/v1/captions/:id" do |env|
  # Auto-generated captions often have cues that aren't aligned properly with the video,
  # as well as some other markup that makes it cumbersome, so we try to fix that here
  if caption.name.simpleText.includes? "auto-generated"
-    caption_xml = client.get(url).body
+    caption_xml = YT_POOL.client &.get(url).body
    caption_xml = XML.parse(caption_xml)

    webvtt = String.build do |str|
@ -3866,7 +3861,7 @@ get "/api/v1/captions/:id" do |env|
      end
    end
  else
-    webvtt = client.get("#{url}&format=vtt").body
+    webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
  end

  if title = env.params.query["title"]?
@ -4013,9 +4008,7 @@ get "/api/v1/annotations/:id" do |env|
      cache_annotation(PG_DB, id, annotations)
    end
  when "youtube"
-    client = make_client(YT_URL)
-
-    response = client.get("/annotations_invideo?video_id=#{id}")
+    response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")

    if response.status_code != 200
      env.response.status_code = response.status_code
@ -5115,7 +5108,6 @@ get "/api/manifest/dash/id/:id" do |env|
  # we can opt to only add a source to a representation if it has a unique height within that representation
  unique_res = env.params.query["unique_res"]? && (env.params.query["unique_res"] == "true" || env.params.query["unique_res"] == "1")

-  client = make_client(YT_URL)
  begin
    video = get_video(id, PG_DB, region: region)
  rescue ex : VideoRedirect
@ -5126,7 +5118,7 @@ get "/api/manifest/dash/id/:id" do |env|
  end

  if dashmpd = video.player_response["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
-    manifest = client.get(dashmpd).body
+    manifest = YT_POOL.client &.get(dashmpd).body

    manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
      url = baseurl.lchop("<BaseURL>")
@ -5226,8 +5218,7 @@ get "/api/manifest/dash/id/:id" do |env|
 end

 get "/api/manifest/hls_variant/*" do |env|
-  client = make_client(YT_URL)
-  manifest = client.get(env.request.path)
+  manifest = YT_POOL.client &.get(env.request.path)

  if manifest.status_code != 200
    env.response.status_code = manifest.status_code
@ -5252,8 +5243,7 @@ get "/api/manifest/hls_variant/*" do |env|
 end

 get "/api/manifest/hls_playlist/*" do |env|
-  client = make_client(YT_URL)
-  manifest = client.get(env.request.path)
+  manifest = YT_POOL.client &.get(env.request.path)

  if manifest.status_code != 200
    env.response.status_code = manifest.status_code
@ -5616,10 +5606,6 @@ get "/videoplayback" do |env|
  end
 end

-# We need this so the below route works as expected
-get "/ggpht*" do |env|
-end
-
 get "/ggpht/*" do |env|
  host = "https://yt3.ggpht.com"
  client = make_client(URI.parse(host))
@ -5745,12 +5731,9 @@ get "/vi/:id/:name" do |env|
  id = env.params.url["id"]
  name = env.params.url["name"]

-  host = "https://i.ytimg.com"
-  client = make_client(URI.parse(host))
-
  if name == "maxres.jpg"
    build_thumbnails(id, config, Kemal.config).each do |thumb|
-      if client.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
+      if YT_IMG_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
        name = thumb[:url] + ".jpg"
        break
      end
@ -5766,7 +5749,7 @@ get "/vi/:id/:name" do |env|
  end

  begin
-    client.get(url, headers) do |response|
+    YT_IMG_POOL.client &.get(url, headers) do |response|
      env.response.status_code = response.status_code
      response.headers.each do |key, value|
        if !RESPONSE_HEADERS_BLACKLIST.includes? key
@ -5789,9 +5772,7 @@ end

 # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
 get "/watch_videos" do |env|
-  client = make_client(YT_URL)
-
-  response = client.get("#{env.request.path}?#{env.request.query}")
+  response = YT_POOL.client &.get(env.request.resource)
  if url = response.headers["Location"]?
    url = URI.parse(url).full_path
    next env.redirect url
@ -5805,11 +5786,10 @@ error 404 do |env|
    item = md["id"]

    # Check if item is branding URL e.g. https://youtube.com/gaming
-    client = make_client(YT_URL)
-    response = client.get("/#{item}")
+    response = YT_POOL.client &.get("/#{item}")

    if response.status_code == 301
-      response = client.get(response.headers["Location"])
+      response = YT_POOL.client &.get(response.headers["Location"])
    end

    if response.body.empty?
@ -5837,8 +5817,7 @@ error 404 do |env|
    end

    # Check if item is video ID
-    client = make_client(YT_URL)
-    if item.match(/^[a-zA-Z0-9_-]{11}$/) && client.head("/watch?v=#{item}").status_code != 404
+    if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
      env.response.headers["Location"] = url
      halt env, status_code: 302
    end
--- a/src/invidious/channels.cr
+++ b/src/invidious/channels.cr
@ -195,9 +195,7 @@ def get_channel(id, db, refresh = true, pull_all_videos = true)
 end

 def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
-  client = make_client(YT_URL)
-
-  rss = client.get("/feeds/videos.xml?channel_id=#{ucid}").body
+  rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
  rss = XML.parse_html(rss)

  author = rss.xpath_node(%q(//feed/title))
@ -216,7 +214,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
  page = 1

  url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
-  response = client.get(url)
+  response = YT_POOL.client &.get(url)
  json = JSON.parse(response.body)

  if json["content_html"]? && !json["content_html"].as_s.empty?
@ -296,7 +294,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)

    loop do
      url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
-      response = client.get(url)
+      response = YT_POOL.client &.get(url)
      json = JSON.parse(response.body)

      if json["content_html"]? && !json["content_html"].as_s.empty?
@ -375,12 +373,10 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
 end

 def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
-  client = make_client(YT_URL)
-
  if continuation
    url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated)

-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
    json = JSON.parse(response.body)

    if json["load_more_widget_html"].as_s.empty?
@ -399,7 +395,7 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
  elsif auto_generated
    url = "/channel/#{ucid}"

-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
    html = XML.parse_html(response.body)

    nodeset = html.xpath_nodes(%q(//ul[@id="browse-items-primary"]/li[contains(@class, "feed-item-container")]))
@ -415,7 +411,7 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
      url += "&sort=dd"
    end

-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
    html = XML.parse_html(response.body)

    continuation = html.xpath_node(%q(//button[@data-uix-load-more-href]))
@ -625,13 +621,12 @@ end

 # TODO: Add "sort_by"
 def fetch_channel_community(ucid, continuation, locale, config, kemal_config, format, thin_mode)
-  client = make_client(YT_URL)
  headers = HTTP::Headers.new
  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"

-  response = client.get("/channel/#{ucid}/community?gl=US&hl=en", headers)
+  response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en", headers)
  if response.status_code == 404
-    response = client.get("/user/#{ucid}/community?gl=US&hl=en", headers)
+    response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en", headers)
  end

  if response.status_code == 404
@ -668,7 +663,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
      session_token: session_token,
    }

-    response = client.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req)
+    response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req)
    body = JSON.parse(response.body)

    body = body["response"]["continuationContents"]["itemSectionContinuation"]? ||
@ -929,11 +924,9 @@ def extract_channel_community_cursor(continuation)
 end

 def get_about_info(ucid, locale)
-  client = make_client(YT_URL)
-
-  about = client.get("/channel/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
+  about = YT_POOL.client &.get("/channel/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
  if about.status_code == 404
-    about = client.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
+    about = YT_POOL.client &.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
  end

  if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/)
@ -1038,11 +1031,9 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
  count = 0
  videos = [] of SearchVideo

-  client = make_client(YT_URL)
-
  2.times do |i|
    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
    json = JSON.parse(response.body)

    if json["content_html"]? && !json["content_html"].as_s.empty?
@ -1067,11 +1058,10 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
 end

 def get_latest_videos(ucid)
-  client = make_client(YT_URL)
  videos = [] of SearchVideo

  url = produce_channel_videos_url(ucid, 0)
-  response = client.get(url)
+  response = YT_POOL.client &.get(url)
  json = JSON.parse(response.body)

  if json["content_html"]? && !json["content_html"].as_s.empty?
--- a/src/invidious/comments.cr
+++ b/src/invidious/comments.cr
@ -85,7 +85,6 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
    session_token: session_token,
  }

-  client = make_client(YT_URL, video.info["region"]?)
  headers = HTTP::Headers.new

  headers["content-type"] = "application/x-www-form-urlencoded"
@ -98,7 +97,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
  headers["x-youtube-client-name"] = "1"
  headers["x-youtube-client-version"] = "2.20180719"

-  response = client.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req)
+  response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req))
  response = JSON.parse(response.body)

  if !response["response"]["continuationContents"]?
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@ -234,6 +234,7 @@ struct Config
    force_resolve:     {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
    port:              {type: Int32, default: 3000},                                                        # Port to listen for connections (overrided by command line argument)
    host_binding:      {type: String, default: "0.0.0.0"},                                                  # Host to bind (overrided by command line argument)
+    pool_size:         {type: Int32, default: 100},
  })
 end

--- a/src/invidious/helpers/proxy.cr
+++ b/src/invidious/helpers/proxy.cr
@ -77,6 +77,10 @@ class HTTPClient < HTTP::Client
    end
  end

+  def unset_proxy
+    @socket = nil
+  end
+
  def proxy_connection_options
    opts = {} of Symbol => Float64 | Nil

--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@ -1,8 +1,7 @@
 def fetch_decrypt_function(id = "CvFH_6DNRCY")
-  client = make_client(YT_URL)
-  document = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
+  document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
  url = document.match(/src="(?<url>\/yts\/jsbin\/player_ias-.{9}\/en_US\/base.js)"/).not_nil!["url"]
-  player = client.get(url).body
+  player = YT_POOL.client &.get(url).body

  function_name = player.match(/^(?<name>[^=]+)=function\(a\){a=a\.split\(""\)/m).not_nil!["name"]
  function_body = player.match(/^#{Regex.escape(function_name)}=function\(a\){(?<body>[^}]+)}/m).not_nil!["body"]
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@ -1,3 +1,46 @@
+require "pool/connection"
+
+struct HTTPPool
+  property! url : URI
+  property! capacity : Int32
+  property! timeout : Float64
+  property pool : ConnectionPool(HTTPClient)
+
+  def initialize(url : URI, @capacity = 5, @timeout = 5.0)
+    @url = url
+    @pool = build_pool
+  end
+
+  def client(region = nil, &block)
+    pool.connection do |conn|
+      if region
+        PROXY_LIST[region]?.try &.sample(40).each do |proxy|
+          begin
+            proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
+            conn.set_proxy(proxy)
+            break
+          rescue ex
+          end
+        end
+      end
+
+      response = yield conn
+      conn.unset_proxy
+      response
+    end
+  end
+
+  private def build_pool
+    ConnectionPool(HTTPClient).new(capacity: capacity, timeout: timeout) do
+      client = HTTPClient.new(url)
+      client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
+      client.read_timeout = 5.seconds
+      client.connect_timeout = 5.seconds
+      client
+    end
+  end
+end
+
 # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
 def ci_lower_bound(pos, n)
  if n == 0
@ -21,8 +64,8 @@ end
 def make_client(url : URI, region = nil)
  client = HTTPClient.new(url)
  client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
-  client.read_timeout = 15.seconds
-  client.connect_timeout = 15.seconds
+  client.read_timeout = 5.seconds
+  client.connect_timeout = 5.seconds

  if region
    PROXY_LIST[region]?.try &.sample(40).each do |proxy|
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@ -19,14 +19,13 @@ struct Mix
 end

 def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
-  client = make_client(YT_URL)
  headers = HTTP::Headers.new
  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"

  if cookies
    headers = cookies.add_request_headers(headers)
  end
-  response = client.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)
+  response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)

  initial_data = extract_initial_data(response.body)

--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@ -384,13 +384,11 @@ def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
 end

 def fetch_playlist(plid, locale)
-  client = make_client(YT_URL)
-
  if plid.starts_with? "UC"
    plid = "UU#{plid.lchop("UC")}"
  end

-  response = client.get("/playlist?list=#{plid}&hl=en&disable_polymer=1")
+  response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en&disable_polymer=1")
  if response.status_code != 200
    raise translate(locale, "Not a playlist.")
  end
@ -458,10 +456,8 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
 end

 def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil)
-  client = make_client(YT_URL)
-
  if continuation
-    html = client.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+    html = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
    html = XML.parse_html(html.body)

    index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1
@ -471,7 +467,7 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
  if video_count > 100
    url = produce_playlist_url(plid, offset)

-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
    response = JSON.parse(response.body)
    if !response["content_html"]? || response["content_html"].as_s.empty?
      raise translate(locale, "Empty playlist")
@ -483,7 +479,7 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
  elsif offset > 100
    return [] of PlaylistVideo
  else # Extract first page of videos
-    response = client.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
+    response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
    document = XML.parse_html(response.body)
    nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))

--- a/src/invidious/search.cr
+++ b/src/invidious/search.cr
@ -222,20 +222,18 @@ end
 alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist

 def channel_search(query, page, channel)
-  client = make_client(YT_URL)
-
-  response = client.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
+  response = YT_POOL.client &.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
  document = XML.parse_html(response.body)
  canonical = document.xpath_node(%q(//link[@rel="canonical"]))

  if !canonical
-    response = client.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
+    response = YT_POOL.client &.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
    document = XML.parse_html(response.body)
    canonical = document.xpath_node(%q(//link[@rel="canonical"]))
  end

  if !canonical
-    response = client.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
+    response = YT_POOL.client &.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
    document = XML.parse_html(response.body)
    canonical = document.xpath_node(%q(//link[@rel="canonical"]))
  end
@ -247,7 +245,7 @@ def channel_search(query, page, channel)
  ucid = canonical["href"].split("/")[-1]

  url = produce_channel_search_url(ucid, query, page)
-  response = client.get(url)
+  response = YT_POOL.client &.get(url)
  json = JSON.parse(response.body)

  if json["content_html"]? && !json["content_html"].as_s.empty?
@ -265,12 +263,11 @@ def channel_search(query, page, channel)
 end

 def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil)
-  client = make_client(YT_URL, region)
  if query.empty?
    return {0, [] of SearchItem}
  end

-  html = client.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body
+  html = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body)
  if html.empty?
    return {0, [] of SearchItem}
  end
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@ -1,5 +1,4 @@
 def fetch_trending(trending_type, region, locale)
-  client = make_client(YT_URL)
  headers = HTTP::Headers.new
  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"

@ -12,7 +11,7 @@ def fetch_trending(trending_type, region, locale)
  if trending_type && trending_type != "Default"
    trending_type = trending_type.downcase.capitalize

-    response = client.get("/feed/trending?gl=#{region}&hl=en", headers).body
+    response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en", headers).body

    initial_data = extract_initial_data(response)

@ -23,13 +22,13 @@ def fetch_trending(trending_type, region, locale)
      url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
      url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
      url += "&disable_polymer=1&gl=#{region}&hl=en"
-      trending = client.get(url).body
+      trending = YT_POOL.client &.get(url).body
      plid = extract_plid(url)
    else
-      trending = client.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+      trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
    end
  else
-    trending = client.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+    trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
  end

  trending = XML.parse_html(trending)
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@ -143,8 +143,7 @@ def get_user(sid, headers, db, refresh = true)
 end

 def fetch_user(sid, headers, db)
-  client = make_client(YT_URL)
-  feed = client.get("/subscription_manager?disable_polymer=1", headers)
+  feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
  feed = XML.parse_html(feed.body)

  channels = [] of String
@ -254,8 +253,7 @@ def subscribe_ajax(channel_id, action, env_headers)
  headers = HTTP::Headers.new
  headers["Cookie"] = env_headers["Cookie"]

-  client = make_client(YT_URL)
-  html = client.get("/subscription_manager?disable_polymer=1", headers)
+  html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)

  cookies = HTTP::Cookies.from_headers(headers)
  html.cookies.each do |cookie|
@ -279,7 +277,7 @@ def subscribe_ajax(channel_id, action, env_headers)
    }
    post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"

-    client.post(post_url, headers, form: post_req)
+    YT_POOL.client &.post(post_url, headers, form: post_req)
  end
 end

@ -288,8 +286,7 @@ end
 #   headers = HTTP::Headers.new
 #   headers["Cookie"] = env_headers["Cookie"]
 #
-#   client = make_client(YT_URL)
-#   html = client.get("/view_all_playlists?disable_polymer=1", headers)
+#   html = YT_POOL.client &.get("/view_all_playlists?disable_polymer=1", headers)
 #
 #   cookies = HTTP::Cookies.from_headers(headers)
 #   html.cookies.each do |cookie|
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@ -1146,8 +1146,7 @@ def extract_player_config(body, html)
 end

 def fetch_video(id, region)
-  client = make_client(YT_URL, region)
-  response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+  response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))

  if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/)
    raise VideoRedirect.new(video_id: md["id"])
@ -1167,8 +1166,7 @@ def fetch_video(id, region)
    bypass_regions = PROXY_LIST.keys & allowed_regions
    if !bypass_regions.empty?
      region = bypass_regions[rand(bypass_regions.size)]
-      client = make_client(YT_URL, region)
-      response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+      response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))

      html = XML.parse_html(response.body)
      info = extract_player_config(response.body, html)
@ -1180,10 +1178,10 @@ def fetch_video(id, region)

  # Try to pull streams from embed URL
  if info["reason"]?
-    embed_page = client.get("/embed/#{id}").body
+    embed_page = YT_POOL.client &.get("/embed/#{id}").body
    sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]?
    sts ||= ""
-    embed_info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body)
+    embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body)

    if !embed_info["reason"]?
      embed_info.each do |key, value|