CAPTION_LANGUAGES = { "", "English", "English (auto-generated)", "Afrikaans", "Albanian", "Amharic", "Arabic", "Armenian", "Azerbaijani", "Bangla", "Basque", "Belarusian", "Bosnian", "Bulgarian", "Burmese", "Catalan", "Cebuano", "Chinese (Simplified)", "Chinese (Traditional)", "Corsican", "Croatian", "Czech", "Danish", "Dutch", "Esperanto", "Estonian", "Filipino", "Finnish", "French", "Galician", "Georgian", "German", "Greek", "Gujarati", "Haitian Creole", "Hausa", "Hawaiian", "Hebrew", "Hindi", "Hmong", "Hungarian", "Icelandic", "Igbo", "Indonesian", "Irish", "Italian", "Japanese", "Javanese", "Kannada", "Kazakh", "Khmer", "Korean", "Kurdish", "Kyrgyz", "Lao", "Latin", "Latvian", "Lithuanian", "Luxembourgish", "Macedonian", "Malagasy", "Malay", "Malayalam", "Maltese", "Maori", "Marathi", "Mongolian", "Nepali", "Norwegian", "Nyanja", "Pashto", "Persian", "Polish", "Portuguese", "Punjabi", "Romanian", "Russian", "Samoan", "Scottish Gaelic", "Serbian", "Shona", "Sindhi", "Sinhala", "Slovak", "Slovenian", "Somali", "Southern Sotho", "Spanish", "Spanish (Latin America)", "Sundanese", "Swahili", "Swedish", "Tajik", "Tamil", "Telugu", "Thai", "Turkish", "Ukrainian", "Urdu", "Uzbek", "Vietnamese", "Welsh", "Western Frisian", "Xhosa", "Yiddish", "Yoruba", "Zulu", } REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} BYPASS_REGIONS = { "GB", "DE", "FR", "IN", "CN", "RU", "CA", "JP", "IT", "TH", "ES", "AE", "KR", "IR", "BR", "PK", "ID", "BD", "MX", "PH", "EG", "VN", "CD", "TR", } # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 VIDEO_FORMATS = { "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, # 3D videos "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, # Apple HTTP Live Streaming "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, # DASH mp4 video "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https=>//github.com/rg3/youtube-dl/issues/4559) "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, # Dash mp4 audio "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, # Dash webm "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, # Dash webm audio "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, # Dash webm audio with opus inside "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, } class Video property player_json : JSON::Any? module HTTPParamConverter def self.from_rs(rs) HTTP::Params.parse(rs.read(String)) end end def keywords keywords = self.player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a keywords ||= [] of String return keywords end def fmt_stream(decrypt_function) streams = [] of HTTP::Params if fmt_streams = self.player_response["streamingData"]?.try &.["formats"]? fmt_streams.as_a.each do |fmt_stream| if !fmt_stream.as_h? next end fmt = {} of String => String fmt["lmt"] = fmt_stream["lastModified"]?.try &.as_s || "0" fmt["projection_type"] = "1" fmt["type"] = fmt_stream["mimeType"].as_s fmt["clen"] = fmt_stream["contentLength"]?.try &.as_s || "0" fmt["bitrate"] = fmt_stream["bitrate"]?.try &.as_i.to_s || "0" fmt["itag"] = fmt_stream["itag"].as_i.to_s fmt["url"] = fmt_stream["url"].as_s fmt["quality"] = fmt_stream["quality"].as_s if fmt_stream["width"]? fmt["size"] = "#{fmt_stream["width"]}x#{fmt_stream["height"]}" fmt["height"] = fmt_stream["height"].as_i.to_s end if fmt_stream["fps"]? fmt["fps"] = fmt_stream["fps"].as_i.to_s end if fmt_stream["qualityLabel"]? fmt["quality_label"] = fmt_stream["qualityLabel"].as_s end params = HTTP::Params.new fmt.each do |key, value| params[key] = value end streams << params end streams.sort_by! { |stream| stream["height"].to_i }.reverse! elsif fmt_stream = self.info["url_encoded_fmt_stream_map"]? fmt_stream.split(",").each do |string| if !string.empty? streams << HTTP::Params.parse(string) end end end streams.each { |s| s.add("label", "#{s["quality"]} - #{s["type"].split(";")[0].split("/")[1]}") } streams = streams.uniq { |s| s["label"] } if self.info["region"]? streams.each do |fmt| fmt["url"] += "®ion=" + self.info["region"] end end streams.each do |fmt| fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "") fmt["url"] += decrypt_signature(fmt, decrypt_function) end return streams end def adaptive_fmts(decrypt_function) adaptive_fmts = [] of HTTP::Params if fmts = self.player_response["streamingData"]?.try &.["adaptiveFormats"]? fmts.as_a.each do |adaptive_fmt| if !adaptive_fmt.as_h? next end fmt = {} of String => String if init = adaptive_fmt["initRange"]? fmt["init"] = "#{init["start"]}-#{init["end"]}" end fmt["init"] ||= "0-0" fmt["lmt"] = adaptive_fmt["lastModified"]?.try &.as_s || "0" fmt["projection_type"] = "1" fmt["type"] = adaptive_fmt["mimeType"].as_s fmt["clen"] = adaptive_fmt["contentLength"]?.try &.as_s || "0" fmt["bitrate"] = adaptive_fmt["bitrate"]?.try &.as_i.to_s || "0" fmt["itag"] = adaptive_fmt["itag"].as_i.to_s fmt["url"] = adaptive_fmt["url"].as_s if index = adaptive_fmt["indexRange"]? fmt["index"] = "#{index["start"]}-#{index["end"]}" end fmt["index"] ||= "0-0" if adaptive_fmt["width"]? fmt["size"] = "#{adaptive_fmt["width"]}x#{adaptive_fmt["height"]}" end if adaptive_fmt["fps"]? fmt["fps"] = adaptive_fmt["fps"].as_i.to_s end if adaptive_fmt["qualityLabel"]? fmt["quality_label"] = adaptive_fmt["qualityLabel"].as_s end params = HTTP::Params.new fmt.each do |key, value| params[key] = value end adaptive_fmts << params end elsif fmts = self.info["adaptive_fmts"]? fmts.split(",") do |string| adaptive_fmts << HTTP::Params.parse(string) end end if self.info["region"]? adaptive_fmts.each do |fmt| fmt["url"] += "®ion=" + self.info["region"] end end adaptive_fmts.each do |fmt| fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "") fmt["url"] += decrypt_signature(fmt, decrypt_function) end return adaptive_fmts end def video_streams(adaptive_fmts) video_streams = adaptive_fmts.select { |s| s["type"].starts_with? "video" } return video_streams end def audio_streams(adaptive_fmts) audio_streams = adaptive_fmts.select { |s| s["type"].starts_with? "audio" } audio_streams.sort_by! { |s| s["bitrate"].to_i }.reverse! audio_streams.each do |stream| stream["bitrate"] = (stream["bitrate"].to_f64/1000).to_i.to_s end return audio_streams end def player_response if !@player_json @player_json = JSON.parse(@info["player_response"]) end return @player_json.not_nil! end def paid reason = self.player_response["playabilityStatus"]?.try &.["reason"]? if reason == "This video requires payment to watch." paid = true else paid = false end return paid end def premium premium = self.player_response.to_s.includes? "Get YouTube without the ads." return premium end def captions captions = [] of Caption if player_response["captions"]? caption_list = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a caption_list ||= [] of JSON::Any caption_list.each do |caption| caption = Caption.from_json(caption.to_json) caption.name.simpleText = caption.name.simpleText.split(" - ")[0] captions << caption end end return captions end def short_description description = self.description.gsub("
", " ") description = description.gsub("
", " ") description = XML.parse_html(description).content[0..200].gsub('"', """).gsub("\n", " ").strip(" ") if description.empty? description = " " end return description end def length_seconds return self.info["length_seconds"].to_i end add_mapping({ id: String, info: { type: HTTP::Params, default: HTTP::Params.parse(""), converter: Video::HTTPParamConverter, }, updated: Time, title: String, views: Int64, likes: Int32, dislikes: Int32, wilson_score: Float64, published: Time, description: String, language: String?, author: String, ucid: String, allowed_regions: Array(String), is_family_friendly: Bool, genre: String, genre_url: String, license: String, sub_count_text: String, author_thumbnail: String, }) end class Caption JSON.mapping( name: CaptionName, baseUrl: String, languageCode: String ) end class CaptionName JSON.mapping( simpleText: String, ) end class VideoRedirect < Exception end def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true, region = nil) if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) && !region video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours) if refresh && Time.now - video.updated > 10.minutes begin video = fetch_video(id, proxies, region) video_array = video.to_a args = arg_array(video_array[1..-1], 2) db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\ published,description,language,author,ucid,allowed_regions,is_family_friendly,\ genre,genre_url,license,sub_count_text,author_thumbnail)\ = (#{args}) WHERE id = $1", video_array) rescue ex db.exec("DELETE FROM videos * WHERE id = $1", id) raise ex end end else video = fetch_video(id, proxies, region) video_array = video.to_a args = arg_array(video_array) if !region db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array) end end return video end def extract_player_config(body, html) params = HTTP::Params.new if md = body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/) params["session_token"] = md["session_token"] end if md = body.match(/itct=(?[^"]+)"/) params["itct"] = md["itct"] end if md = body.match(/'COMMENTS_TOKEN': "(?[^"]+)"/) params["ctoken"] = md["ctoken"] end if md = body.match(/'RELATED_PLAYER_ARGS': (?{"rvs":"[^"]+"})/) params["rvs"] = JSON.parse(md["rvs"])["rvs"].as_s end html_info = body.match(/ytplayer\.config = (?.*?);ytplayer\.load/).try &.["info"] if html_info JSON.parse(html_info)["args"].as_h.each do |key, value| params[key] = value.to_s end else error_message = html.xpath_node(%q(//h1[@id="unavailable-message"])) if error_message params["reason"] = error_message.content.strip else params["reason"] = "Could not extract video info." end end return params end def fetch_video(id, proxies, region) client = make_client(YT_URL, proxies, region) response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") if md = response.headers["location"]?.try &.match(/v=(?[a-zA-Z0-9_-]{11})/) raise VideoRedirect.new(md["id"]) end html = XML.parse_html(response.body) info = extract_player_config(response.body, html) info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ") # Try to use proxies for region-blocked videos if info["reason"]? && info["reason"].includes? "your country" bypass_channel = Channel({XML::Node, HTTP::Params} | Nil).new proxies.each do |proxy_region, list| spawn do client = make_client(YT_URL, proxies, proxy_region) proxy_response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") proxy_html = XML.parse_html(proxy_response.body) proxy_info = extract_player_config(proxy_response.body, proxy_html) if !proxy_info["reason"]? proxy_info["region"] = proxy_region proxy_info["cookie"] = proxy_response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ") bypass_channel.send({proxy_html, proxy_info}) else bypass_channel.send(nil) end end end proxies.size.times do response = bypass_channel.receive if response html, info = response break end end end # Try to pull streams from embed URL if info["reason"]? embed_page = client.get("/embed/#{id}").body sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]? sts ||= "" embed_info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body) if !embed_info["reason"]? embed_info.each do |key, value| info[key] = value.to_s end else raise info["reason"] end end if info["errorcode"]?.try &.== "2" raise "Video unavailable." end title = info["title"] author = info["author"] ucid = info["ucid"] views = html.xpath_node(%q(//meta[@itemprop="interactionCount"])) views = views.try &.["content"].to_i64? views ||= 0_i64 likes = html.xpath_node(%q(//button[@title="I like this"]/span)) likes = likes.try &.content.delete(",").try &.to_i? likes ||= 0 dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span)) dislikes = dislikes.try &.content.delete(",").try &.to_i? dislikes ||= 0 avg_rating = (likes.to_f/(likes.to_f + dislikes.to_f) * 4 + 1) avg_rating = avg_rating.nan? ? 0.0 : avg_rating info["avg_rating"] = "#{avg_rating}" description = html.xpath_node(%q(//p[@id="eow-description"])) description = description ? description.to_xml : "" wilson_score = ci_lower_bound(likes, likes + dislikes) published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"] published ||= Time.now.to_s("%Y-%m-%d") published = Time.parse(published, "%Y-%m-%d", Time::Location.local) allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",") allowed_regions ||= [] of String is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True" is_family_friendly ||= true genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"] genre ||= "" genre_url = html.xpath_node(%(//ul[contains(@class, "watch-info-tag-list")]/li/a[text()="#{genre}"])).try &.["href"] # Sometimes YouTube tries to link to invalid/missing channels, so we fix that here case genre when "Education" genre_url = "/channel/UCdxpofrI-dO6oYfsqHDHphw" when "Gaming" genre_url = "/channel/UCOpNcN46UbXVtpKMrmU4Abg" when "Movies" genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g" when "Nonprofits & Activism" genre_url = "/channel/UCfFyYRYslvuhwMDnx6KjUvw" when "Trailers" genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g" end genre_url ||= "" license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)) if license license = license.content else license = "" end sub_count_text = html.xpath_node(%q(//span[contains(@class, "yt-subscriber-count")])) if sub_count_text sub_count_text = sub_count_text["title"] else sub_count_text = "0" end author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)) if author_thumbnail author_thumbnail = author_thumbnail["data-thumb"] else author_thumbnail = "" end video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description, nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail) return video end def itag_to_metadata?(itag : String) return VIDEO_FORMATS[itag]? end def process_video_params(query, preferences) autoplay = query["autoplay"]?.try &.to_i? continue = query["continue"]?.try &.to_i? listen = query["listen"]? && (query["listen"] == "true" || query["listen"] == "1").to_unsafe local = query["local"]? && (query["local"] == "true").to_unsafe preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? region = query["region"]? related_videos = query["related_videos"]? speed = query["speed"]?.try &.to_f? video_loop = query["loop"]?.try &.to_i? volume = query["volume"]?.try &.to_i? if preferences # region ||= preferences.region autoplay ||= preferences.autoplay.to_unsafe continue ||= preferences.continue.to_unsafe related_videos ||= preferences.related_videos.to_unsafe listen ||= preferences.listen.to_unsafe preferred_captions ||= preferences.captions quality ||= preferences.quality speed ||= preferences.speed video_loop ||= preferences.video_loop.to_unsafe volume ||= preferences.volume end autoplay ||= DEFAULT_USER_PREFERENCES.autoplay.to_unsafe continue ||= DEFAULT_USER_PREFERENCES.continue.to_unsafe related_videos ||= DEFAULT_USER_PREFERENCES.related_videos.to_unsafe listen ||= DEFAULT_USER_PREFERENCES.listen.to_unsafe preferred_captions ||= DEFAULT_USER_PREFERENCES.captions quality ||= DEFAULT_USER_PREFERENCES.quality speed ||= DEFAULT_USER_PREFERENCES.speed video_loop ||= DEFAULT_USER_PREFERENCES.video_loop.to_unsafe volume ||= DEFAULT_USER_PREFERENCES.volume autoplay = autoplay == 1 continue = continue == 1 listen = listen == 1 local = local == 1 related_videos = related_videos == 1 video_loop = video_loop == 1 if query["t"]? video_start = decode_time(query["t"]) end video_start ||= 0 if query["time_continue"]? video_start = decode_time(query["time_continue"]) end video_start ||= 0 if query["start"]? video_start = decode_time(query["start"]) end if query["end"]? video_end = decode_time(query["end"]) end video_end ||= -1 raw = query["raw"]?.try &.to_i? raw ||= 0 raw = raw == 1 controls = query["controls"]?.try &.to_i? controls ||= 1 controls = controls == 1 params = { autoplay: autoplay, continue: continue, controls: controls, listen: listen, local: local, preferred_captions: preferred_captions, quality: quality, raw: raw, region: region, related_videos: related_videos, speed: speed, video_end: video_end, video_loop: video_loop, video_start: video_start, volume: volume, } return params end def build_thumbnails(id, config, kemal_config) return { {name: "maxres", host: "#{make_host_url(config, kemal_config)}", url: "maxres", height: 720, width: 1280}, {name: "maxresdefault", host: "https://i.ytimg.com", url: "maxresdefault", height: 720, width: 1280}, {name: "sddefault", host: "https://i.ytimg.com", url: "sddefault", height: 480, width: 640}, {name: "high", host: "https://i.ytimg.com", url: "hqdefault", height: 360, width: 480}, {name: "medium", host: "https://i.ytimg.com", url: "mqdefault", height: 180, width: 320}, {name: "default", host: "https://i.ytimg.com", url: "default", height: 90, width: 120}, {name: "start", host: "https://i.ytimg.com", url: "1", height: 90, width: 120}, {name: "middle", host: "https://i.ytimg.com", url: "2", height: 90, width: 120}, {name: "end", host: "https://i.ytimg.com", url: "3", height: 90, width: 120}, } end def generate_thumbnails(json, id, config, kemal_config) json.array do build_thumbnails(id, config, kemal_config).each do |thumbnail| json.object do json.field "quality", thumbnail[:name] json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" json.field "width", thumbnail[:width] json.field "height", thumbnail[:height] end end end end