From 19516eaa25f5d2ba723b2c28adaa40b745589880 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 31 Oct 2018 16:47:53 -0500 Subject: [PATCH] Add option to view comments with JS disabled --- src/invidious.cr | 266 ++++++++-------------------------- src/invidious/comments.cr | 216 ++++++++++++++++++++++++++- src/invidious/views/watch.ecr | 8 + 3 files changed, 281 insertions(+), 209 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 8fcb66e3a..0ad09e2ff 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -229,6 +229,10 @@ get "/watch" do |env| end plid = env.params.query["list"]? + nojs = env.params.query["nojs"]? + + nojs ||= "0" + nojs = nojs == "1" user = env.get? "user" if user @@ -255,6 +259,51 @@ get "/watch" do |env| next templated "error" end + if nojs + if preferences + source = preferences.comments[0] + if source.empty? + source = preferences.comments[1] + end + + if source == "youtube" + begin + comments = fetch_youtube_comments(id, "", proxies, "html") + comments = JSON.parse(comments) + comment_html = template_youtube_comments(comments) + rescue ex + if preferences.comments[1] == "reddit" + comments, reddit_thread = fetch_reddit_comments(id) + comment_html = template_reddit_comments(comments) + + comment_html = fill_links(comment_html, "https", "www.reddit.com") + comment_html = replace_links(comment_html) + end + end + elsif source == "reddit" + begin + comments, reddit_thread = fetch_reddit_comments(id) + comment_html = template_reddit_comments(comments) + + comment_html = fill_links(comment_html, "https", "www.reddit.com") + comment_html = replace_links(comment_html) + rescue ex + if preferences.comments[1] == "youtube" + comments = fetch_youtube_comments(id, "", proxies, "html") + comments = JSON.parse(comments) + comment_html = template_youtube_comments(comments) + end + end + end + else + comments = fetch_youtube_comments(id, "", proxies, "html") + comments = JSON.parse(comments) + comment_html = template_youtube_comments(comments) + end + + comment_html ||= "" + end + fmt_stream = video.fmt_stream(decrypt_function) adaptive_fmts = video.adaptive_fmts(decrypt_function) video_streams = video.video_streams(adaptive_fmts) @@ -1863,212 +1912,15 @@ get "/api/v1/comments/:id" do |env| format = env.params.query["format"]? format ||= "json" + continuation = env.params.query["continuation"]? + continuation ||= "" + if source == "youtube" - client = make_client(YT_URL) - html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") - headers = HTTP::Headers.new - headers["cookie"] = html.cookies.add_request_headers(headers)["cookie"] - body = html.body - - session_token = body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] - itct = body.match(/itct=(?[^"]+)"/).not_nil!["itct"] - ctoken = body.match(/'COMMENTS_TOKEN': "(?[^"]+)"/) - - if body.match(//) - bypass_channel = Channel({String, HTTPClient, HTTP::Headers} | Nil).new - - proxies.each do |region, list| - spawn do - proxy_html = %() - - list.each do |proxy| - begin - proxy_client = HTTPClient.new(YT_URL) - proxy_client.read_timeout = 10.seconds - proxy_client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - proxy_client.set_proxy(proxy) - - response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") - proxy_headers = HTTP::Headers.new - proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] - proxy_html = response.body - - if !proxy_html.match(//) - bypass_channel.send({proxy_html, proxy_client, proxy_headers}) - break - end - rescue ex - end - end - - # If none of the proxies we tried returned a valid response - if proxy_html.match(//) - bypass_channel.send(nil) - end - end - end - - proxies.size.times do - response = bypass_channel.receive - if response - session_token = response[0].match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] - itct = response[0].match(/itct=(?[^"]+)"/).not_nil!["itct"] - ctoken = response[0].match(/'COMMENTS_TOKEN': "(?[^"]+)"/) - - client = response[1] - headers = response[2] - break - end - end - end - - if !ctoken - if format == "json" - next {"comments" => [] of String}.to_json - else - next {"contentHtml" => "", "commentCount" => 0}.to_json - end - end - ctoken = ctoken["ctoken"] - - if env.params.query["continuation"]? && !env.params.query["continuation"].empty? - continuation = env.params.query["continuation"] - ctoken = continuation - else - continuation = ctoken - end - - post_req = { - "session_token" => session_token, - } - post_req = HTTP::Params.encode(post_req) - - headers["content-type"] = "application/x-www-form-urlencoded" - - headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ==" - headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1" - headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1" - - headers["x-youtube-client-name"] = "1" - headers["x-youtube-client-version"] = "2.20180719" - response = client.post("/comment_service_ajax?action_get_comments=1&pbj=1&ctoken=#{ctoken}&continuation=#{continuation}&itct=#{itct}&hl=en&gl=US", headers, post_req) - response = JSON.parse(response.body) - - if !response["response"]["continuationContents"]? - halt env, status_code: 500 - end - - response = response["response"]["continuationContents"] - if response["commentRepliesContinuation"]? - body = response["commentRepliesContinuation"] - else - body = response["itemSectionContinuation"] - end - contents = body["contents"]? - if !contents - if format == "json" - next {"comments" => [] of String}.to_json - else - next {"contentHtml" => "", "commentCount" => 0}.to_json - end - end - - comments = JSON.build do |json| - json.object do - if body["header"]? - comment_count = body["header"]["commentsHeaderRenderer"]["countText"]["simpleText"].as_s.delete("Comments,").to_i - json.field "commentCount", comment_count - end - - json.field "comments" do - json.array do - contents.as_a.each do |node| - json.object do - if !response["commentRepliesContinuation"]? - node = node["commentThreadRenderer"] - end - - if node["replies"]? - node_replies = node["replies"]["commentRepliesRenderer"] - end - - if !response["commentRepliesContinuation"]? - node_comment = node["comment"]["commentRenderer"] - else - node_comment = node["commentRenderer"] - end - - content_html = node_comment["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff') - if content_html - content_html = HTML.escape(content_html) - end - - content_html ||= content_to_comment_html(node_comment["contentText"]["runs"].as_a) - content_html, content = html_to_content(content_html) - - author = node_comment["authorText"]?.try &.["simpleText"] - author ||= "" - - json.field "author", author - json.field "authorThumbnails" do - json.array do - node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| - json.object do - json.field "url", thumbnail["url"] - json.field "width", thumbnail["width"] - json.field "height", thumbnail["height"] - end - end - end - end - - if node_comment["authorEndpoint"]? - json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] - else - json.field "authorId", "" - json.field "authorUrl", "" - end - - published = decode_date(node_comment["publishedTimeText"]["runs"][0]["text"].as_s.rchop(" (edited)")) - - json.field "content", content - json.field "contentHtml", content_html - json.field "published", published.epoch - json.field "publishedText", "#{recode_date(published)} ago" - json.field "likeCount", node_comment["likeCount"] - json.field "commentId", node_comment["commentId"] - - if node_replies && !response["commentRepliesContinuation"]? - reply_count = node_replies["moreText"]["simpleText"].as_s.delete("View all reply replies,") - if reply_count.empty? - reply_count = 1 - else - reply_count = reply_count.try &.to_i? - reply_count ||= 1 - end - - continuation = node_replies["continuations"].as_a[0]["nextContinuationData"]["continuation"].as_s - - json.field "replies" do - json.object do - json.field "replyCount", reply_count - json.field "continuation", continuation - end - end - end - end - end - end - end - - if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"] - json.field "continuation", continuation - end - end + begin + comments = fetch_youtube_comments(id, continuation, proxies, format) + rescue ex + error_message = {"error" => ex.message}.to_json + halt env, status_code: 500, response: error_message end if format == "json" @@ -2092,10 +1944,8 @@ get "/api/v1/comments/:id" do |env| next response end elsif source == "reddit" - client = make_client(REDDIT_URL) - headers = HTTP::Headers{"User-Agent" => "web:invidio.us:v0.6.0 (by /u/omarroth)"} begin - comments, reddit_thread = get_reddit_comments(id, client, headers) + comments, reddit_thread = fetch_reddit_comments(id) content_html = template_reddit_comments(comments) content_html = fill_links(content_html, "https", "www.reddit.com") diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 09eb4fedf..94c4698ed 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -56,7 +56,221 @@ class RedditListing }) end -def get_reddit_comments(id, client, headers) +def fetch_youtube_comments(id, continuation, proxies, format) + client = make_client(YT_URL) + html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") + headers = HTTP::Headers.new + headers["cookie"] = html.cookies.add_request_headers(headers)["cookie"] + body = html.body + + session_token = body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] + itct = body.match(/itct=(?[^"]+)"/).not_nil!["itct"] + ctoken = body.match(/'COMMENTS_TOKEN': "(?[^"]+)"/) + + if body.match(//) + bypass_channel = Channel({String, HTTPClient, HTTP::Headers} | Nil).new + + proxies.each do |region, list| + spawn do + proxy_html = %() + + list.each do |proxy| + begin + proxy_client = HTTPClient.new(YT_URL) + proxy_client.read_timeout = 10.seconds + proxy_client.connect_timeout = 10.seconds + + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + proxy_client.set_proxy(proxy) + + response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") + proxy_headers = HTTP::Headers.new + proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] + proxy_html = response.body + + if !proxy_html.match(//) + bypass_channel.send({proxy_html, proxy_client, proxy_headers}) + break + end + rescue ex + end + end + + # If none of the proxies we tried returned a valid response + if proxy_html.match(//) + bypass_channel.send(nil) + end + end + end + + proxies.size.times do + response = bypass_channel.receive + if response + session_token = response[0].match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] + itct = response[0].match(/itct=(?[^"]+)"/).not_nil!["itct"] + ctoken = response[0].match(/'COMMENTS_TOKEN': "(?[^"]+)"/) + + client = response[1] + headers = response[2] + break + end + end + end + + if !ctoken + if format == "json" + return {"comments" => [] of String}.to_json + else + return {"contentHtml" => "", "commentCount" => 0}.to_json + end + end + ctoken = ctoken["ctoken"] + + if !continuation.empty? + ctoken = continuation + else + continuation = ctoken + end + + post_req = { + "session_token" => session_token, + } + post_req = HTTP::Params.encode(post_req) + + headers["content-type"] = "application/x-www-form-urlencoded" + + headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ==" + headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1" + headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1" + + headers["x-youtube-client-name"] = "1" + headers["x-youtube-client-version"] = "2.20180719" + response = client.post("/comment_service_ajax?action_get_comments=1&pbj=1&ctoken=#{ctoken}&continuation=#{continuation}&itct=#{itct}&hl=en&gl=US", headers, post_req) + response = JSON.parse(response.body) + + if !response["response"]["continuationContents"]? + raise "Could not fetch comments" + end + + response = response["response"]["continuationContents"] + if response["commentRepliesContinuation"]? + body = response["commentRepliesContinuation"] + else + body = response["itemSectionContinuation"] + end + + contents = body["contents"]? + if !contents + if format == "json" + return {"comments" => [] of String}.to_json + else + return {"contentHtml" => "", "commentCount" => 0}.to_json + end + end + + comments = JSON.build do |json| + json.object do + if body["header"]? + comment_count = body["header"]["commentsHeaderRenderer"]["countText"]["simpleText"].as_s.delete("Comments,").to_i + json.field "commentCount", comment_count + end + + json.field "comments" do + json.array do + contents.as_a.each do |node| + json.object do + if !response["commentRepliesContinuation"]? + node = node["commentThreadRenderer"] + end + + if node["replies"]? + node_replies = node["replies"]["commentRepliesRenderer"] + end + + if !response["commentRepliesContinuation"]? + node_comment = node["comment"]["commentRenderer"] + else + node_comment = node["commentRenderer"] + end + + content_html = node_comment["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff') + if content_html + content_html = HTML.escape(content_html) + end + + content_html ||= content_to_comment_html(node_comment["contentText"]["runs"].as_a) + content_html, content = html_to_content(content_html) + + author = node_comment["authorText"]?.try &.["simpleText"] + author ||= "" + + json.field "author", author + json.field "authorThumbnails" do + json.array do + node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end + + if node_comment["authorEndpoint"]? + json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] + else + json.field "authorId", "" + json.field "authorUrl", "" + end + + published = decode_date(node_comment["publishedTimeText"]["runs"][0]["text"].as_s.rchop(" (edited)")) + + json.field "content", content + json.field "contentHtml", content_html + json.field "published", published.epoch + json.field "publishedText", "#{recode_date(published)} ago" + json.field "likeCount", node_comment["likeCount"] + json.field "commentId", node_comment["commentId"] + + if node_replies && !response["commentRepliesContinuation"]? + reply_count = node_replies["moreText"]["simpleText"].as_s.delete("View all reply replies,") + if reply_count.empty? + reply_count = 1 + else + reply_count = reply_count.try &.to_i? + reply_count ||= 1 + end + + continuation = node_replies["continuations"].as_a[0]["nextContinuationData"]["continuation"].as_s + + json.field "replies" do + json.object do + json.field "replyCount", reply_count + json.field "continuation", continuation + end + end + end + end + end + end + end + + if body["continuations"]? + continuation = body["continuations"][0]["nextContinuationData"]["continuation"] + json.field "continuation", continuation + end + end + end + + return comments +end + +def fetch_reddit_comments(id) + client = make_client(REDDIT_URL) + headers = HTTP::Headers{"User-Agent" => "web:invidio.us:v0.11.0 (by /u/omarroth)"} + query = "(url:3D#{id}%20OR%20url:#{id})%20(site:youtube.com%20OR%20site:youtu.be)" search_results = client.get("/search.json?q=#{query}", headers) diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index b286870d0..3cb6328bc 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -121,6 +121,14 @@
+ <% if nojs %> + <%= comment_html %> + <% else %> + + <% end %>