From 71bc9eea28ed6fb22992d774443f64ce1d1551be Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 9 Nov 2019 14:18:19 -0500 Subject: [PATCH] Add support for Anti-Captcha --- src/invidious.cr | 13 ++++++ src/invidious/helpers/helpers.cr | 23 +++++++++++ src/invidious/helpers/jobs.cr | 71 +++++++++++++++++++++++++++++++- src/invidious/helpers/proxy.cr | 1 + 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 90b428f61..06f9e6242 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -212,6 +212,19 @@ spawn do end end +if CONFIG.captcha_key + spawn do + bypass_captcha(CONFIG.captcha_key, logger) do |cookies| + cookies.each do |cookie| + config.cookies << cookie + end + + # Persist cookies between runs + File.write("config/config.yml", config.to_yaml) + end + end +end + connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) spawn do connections = [] of Channel(PQ::Notification) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index fbb359deb..2341d3be5 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -192,6 +192,27 @@ struct Config end end + module StringToCookies + def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) + (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + cookies = HTTP::Cookies.new + node.value.split(";").each do |cookie| + next if cookie.strip.empty? + name, value = cookie.split("=", 2) + cookies << HTTP::Cookie.new(name.strip, value.strip) + end + + cookies + end + end + def disabled?(option) case disabled = CONFIG.disable_proxy when Bool @@ -236,6 +257,8 @@ struct Config host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument) pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports + cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format + captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha }) end diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index 6bca0dae6..0b46cba29 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -228,10 +228,77 @@ def update_decrypt_function yield decrypt_function rescue ex next + ensure + sleep 1.minute + Fiber.yield end + end +end - sleep 1.minute - Fiber.yield +def bypass_captcha(captcha_key, logger) + loop do + begin + response = YT_POOL.client &.get("/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") + if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") + html = XML.parse_html(response.body) + form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! + site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"] + + inputs = {} of String => String + form.xpath_nodes(%(.//input[@name])).map do |node| + inputs[node["name"]] = node["value"] + end + + headers = response.cookies.add_request_headers(HTTP::Headers.new) + + response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { + "clientKey" => CONFIG.captcha_key, + "task" => { + "type" => "NoCaptchaTaskProxyless", + # "type" => "NoCaptchaTask", + "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999", + "websiteKey" => site_key, + # "proxyType" => "http", + # "proxyAddress" => CONFIG.proxy_address, + # "proxyPort" => CONFIG.proxy_port, + # "proxyLogin" => CONFIG.proxy_user, + # "proxyPassword" => CONFIG.proxy_pass, + # "userAgent" => random_user_agent, + }, + }.to_json).body) + + if response["error"]? + raise response["error"].as_s + end + + task_id = response["taskId"].as_i + + loop do + sleep 10.seconds + + response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: { + "clientKey" => CONFIG.captcha_key, + "taskId" => task_id, + }.to_json).body) + + if response["status"]?.try &.== "ready" + break + elsif response["errorId"]?.try &.as_i != 0 + raise response["errorDescription"].as_s + end + end + + inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s + response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) + + yield response.cookies.select { |cookie| cookie.name != "PREF" } + end + rescue ex + logger.puts("Exception: #{ex.message}") + ensure + sleep 1.minute + Fiber.yield + end end end diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index fd998ef71..1bd9b3008 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -99,6 +99,7 @@ class HTTPClient < HTTP::Client request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" request.headers["accept-language"] ||= "en-us,en;q=0.5" + request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" end super