From 80a513baa5e595f62b08d7eed1ac709533fde838 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 24 Nov 2021 01:22:09 +0100 Subject: [PATCH 1/5] Use new techniques to get (dis)likes back --- src/invidious/videos.cr | 108 ++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d3e5800cc..1f9a6bc98 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -877,42 +877,84 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ } ).try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) - primary_results = player_response.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? - .try &.["results"]?.try &.["contents"]? - sentiment_bar = primary_results.try &.as_a.select(&.["videoPrimaryInfoRenderer"]?)[0]? - .try &.["videoPrimaryInfoRenderer"]? - .try &.["sentimentBar"]? - .try &.["sentimentBarRenderer"]? - .try &.["tooltip"]? - .try &.as_s + # Top level elements - likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64} - params["likes"] = JSON::Any.new(likes) - params["dislikes"] = JSON::Any.new(dislikes) + primary_results = player_response + .dig?("contents", "twoColumnWatchNextResults", "results", "results", "contents") - params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select(&.["videoSecondaryInfoRenderer"]?)[0]? - .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]? - .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "
") } || "

") + video_primary_renderer = primary_results + .try &.as_a.find(&.["videoPrimaryInfoRenderer"]?) + .try &.["videoPrimaryInfoRenderer"] - metadata = primary_results.try &.as_a.select(&.["videoSecondaryInfoRenderer"]?)[0]? - .try &.["videoSecondaryInfoRenderer"]? - .try &.["metadataRowContainer"]? - .try &.["metadataRowContainerRenderer"]? - .try &.["rows"]? - .try &.as_a + video_secondary_renderer = primary_results + .try &.as_a.find(&.["videoSecondaryInfoRenderer"]?) + .try &.["videoSecondaryInfoRenderer"] + + # Likes/dislikes + + toplevel_buttons = video_primary_renderer + .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") + + if toplevel_buttons + likes_button = toplevel_buttons.as_a + .find(&.["toggleButtonRenderer"]["defaultIcon"]["iconType"].as_s.== "LIKE") + .try &.["toggleButtonRenderer"] + + if likes_button + likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + likes = likes_txt.as_s.gsub(/\D/, "").to_i64 if likes_txt + + LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") + LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + end + + dislikes_button = toplevel_buttons.as_a + .find(&.["toggleButtonRenderer"]["defaultIcon"]["iconType"].as_s.== "DISLIKE") + .try &.["toggleButtonRenderer"] + + if dislikes_button + dislikes_txt = (dislikes_button["defaultText"] || dislikes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64 if dislikes_txt + + LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"") + LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes + end + end + + if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64) + if rating = player_response.dig?("videoDetails", "averageRating").try &.as_f + dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64 + LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}") + end + end + + params["likes"] = JSON::Any.new(likes || 0_i64) + params["dislikes"] = JSON::Any.new(dislikes || 0_i64) + + # Description + + description_html = video_secondary_renderer.try &.dig?("description", "runs") + .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "
") } + + params["descriptionHtml"] = JSON::Any.new(description_html || "

") + + # Video metadata + + metadata = video_secondary_renderer + .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") + .try &.as_a params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("") params["genreUrl"] = JSON::Any.new(nil) metadata.try &.each do |row| title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s - contents = row["metadataRowRenderer"]? - .try &.["contents"]? - .try &.as_a[0]? + contents = row.dig?("metadataRowRenderer", "contents", 0) if title.try &.== "Category" - contents = contents.try &.["runs"]? - .try &.as_a[0]? + contents = contents.try &.["runs"]?.try &.as_a[0]? params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]? @@ -927,17 +969,19 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ end end - author_info = primary_results.try &.as_a.select(&.["videoSecondaryInfoRenderer"]?)[0]? - .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]? + # Author infos - params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]? - .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]? - .try &.as_s || "") + author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") + author_thumbnail = author_info.try &.dig?("thumbnail", "thumbnails", 0, "url") + + params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "") params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]? - .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-") + .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }.try &.as_s.split(" ", 2)[0] || "-") - params + # Return data + + return params end def get_video(id, db, refresh = true, region = nil, force_refresh = false) From ceb1feb3502367ec41c8ae7e28a0cfbd7a2e2170 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 25 Nov 2021 23:16:50 +0100 Subject: [PATCH 2/5] likes/dislikes: better fallback management '.to_i64?' instead of '.to_i64' returns nil rather than raising an exception when it's done on an empty string. In some rare cases, rating can be equal to 5. In this case, the value of player_response[videoDetails][averageRating] is an Int and not a Float. --- src/invidious/videos.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 1f9a6bc98..d04c6ecbc 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -903,7 +903,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ if likes_button likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) .try &.dig?("accessibility", "accessibilityData", "label") - likes = likes_txt.as_s.gsub(/\D/, "").to_i64 if likes_txt + likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes @@ -916,7 +916,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ if dislikes_button dislikes_txt = (dislikes_button["defaultText"] || dislikes_button["toggledText"]?) .try &.dig?("accessibility", "accessibilityData", "label") - dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64 if dislikes_txt + dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64? if dislikes_txt LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"") LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes @@ -924,7 +924,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ end if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64) - if rating = player_response.dig?("videoDetails", "averageRating").try &.as_f + if rating = player_response.dig?("videoDetails", "averageRating").try { |x| x.as_i64? || x.as_f? } dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64 LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}") end From 91f83952223a6a3b25268955b3fca6c6cf562fac Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 28 Nov 2021 23:37:27 +0100 Subject: [PATCH 3/5] Typo: missing '?' when looking for key in dislikes_button Co-authored-by: Matthew McGarvey --- src/invidious/videos.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d04c6ecbc..c9f6626a1 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -914,7 +914,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ .try &.["toggleButtonRenderer"] if dislikes_button - dislikes_txt = (dislikes_button["defaultText"] || dislikes_button["toggledText"]?) + dislikes_txt = (dislikes_button["defaultText"]? || dislikes_button["toggledText"]?) .try &.dig?("accessibility", "accessibilityData", "label") dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64? if dislikes_txt From 4436359d0783ca8444467603a820c02372be7e9f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 28 Nov 2021 23:44:37 +0100 Subject: [PATCH 4/5] Use dig to get category contents Co-authored-by: Matthew McGarvey --- src/invidious/videos.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index c9f6626a1..db94110b4 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -954,7 +954,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ contents = row.dig?("metadataRowRenderer", "contents", 0) if title.try &.== "Category" - contents = contents.try &.["runs"]?.try &.as_a[0]? + contents = contents.try &.dig?("runs", 0) params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]? From 4aa96ecab9319df15677d16e73351b88ee990050 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 1 Dec 2021 17:32:10 +0100 Subject: [PATCH 5/5] Use 'dig()' in 'find()' statements --- src/invidious/videos.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index db94110b4..e20fb386c 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -897,7 +897,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ if toplevel_buttons likes_button = toplevel_buttons.as_a - .find(&.["toggleButtonRenderer"]["defaultIcon"]["iconType"].as_s.== "LIKE") + .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "LIKE") .try &.["toggleButtonRenderer"] if likes_button @@ -910,7 +910,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ end dislikes_button = toplevel_buttons.as_a - .find(&.["toggleButtonRenderer"]["defaultIcon"]["iconType"].as_s.== "DISLIKE") + .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "DISLIKE") .try &.["toggleButtonRenderer"] if dislikes_button