From cbdba66ef3f770c31a99a16a47925ed37439b543 Mon Sep 17 00:00:00 2001 From: Svallinn <41585298+Svallinn@users.noreply.github.com> Date: Wed, 24 Mar 2021 05:33:45 +0000 Subject: [PATCH 1/5] Use the youtubei API over the legacy one --- src/invidious.cr | 6 +++--- src/invidious/channels.cr | 26 +++++++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 8d579f923..88b9ad853 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1699,7 +1699,7 @@ get "/channel/:ucid" do |env| sort_options = {"last", "oldest", "newest"} sort_by ||= "last" - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) items.uniq! do |item| if item.responds_to?(:title) item.title @@ -1766,7 +1766,7 @@ get "/channel/:ucid/playlists" do |env| next env.redirect "/channel/#{channel.ucid}" end - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) } items.each { |item| item.author = "" } @@ -2467,7 +2467,7 @@ end next error_json(500, ex) end - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) JSON.build do |json| json.object do diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 9a129e1e7..f7aa99e23 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -355,14 +355,19 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) return channel end -def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) - if continuation || auto_generated - url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated) +def fetch_channel_playlists(ucid, author, continuation, sort_by) + if continuation + response_json = request_youtube_api_browse(continuation) + result = JSON.parse(response_json.match(/"continuationItems": (?\[.*\]),/m).try &.["items"] || "{}") - response = YT_POOL.client &.get(url) + return [] of SearchItem, nil if result.size == 0 - continuation = response.body.match(/"continuation":"(?[^"]+)"/).try &.["continuation"]? - initial_data = JSON.parse(response.body).as_a.find(&.["response"]?).try &.as_h + items = [] of SearchItem + result.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| + extract_item(item, author, ucid).try { |t| items << t } + } + + continuation = result.as_a.last["continuationItemRenderer"]?.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s else url = "/channel/#{ucid}/playlists?flow=list&view=1" @@ -377,13 +382,12 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) end response = YT_POOL.client &.get(url) - continuation = response.body.match(/"continuation":"(?[^"]+)"/).try &.["continuation"]? initial_data = extract_initial_data(response.body) - end + return [] of SearchItem, nil if !initial_data - return [] of SearchItem, nil if !initial_data - items = extract_items(initial_data) - continuation = extract_channel_playlists_cursor(continuation, auto_generated) if continuation + items = extract_items(initial_data, author, ucid) + continuation = response.body.match(/"token":"(?[^"]+)"/).try &.["continuation"]? + end return items, continuation end From aa4c623a06ee6c1bed864a92322876d1503a6857 Mon Sep 17 00:00:00 2001 From: Svallinn <41585298+Svallinn@users.noreply.github.com> Date: Wed, 24 Mar 2021 05:34:23 +0000 Subject: [PATCH 2/5] Add deprecation note --- src/invidious/channels.cr | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index f7aa99e23..3a9b86410 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -457,6 +457,15 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end +# ## NOTE: DEPRECATED +# Reason -> Unstable +# The Protobuf object must be provided with an id of the last playlist from the current "page" +# in order to fetch the next one accurately +# (if the id isn't included, entries shift around erratically between pages, +# leading to repetitions and skip overs) +# +# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user, +# it's better to stick to continuation tokens provided by the first request and onward def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false) object = { "80226972:embedded" => { From e248e7ebaf2ec32d790f7e88a96421cc2d418c9d Mon Sep 17 00:00:00 2001 From: Svallinn <41585298+Svallinn@users.noreply.github.com> Date: Wed, 24 Mar 2021 05:35:26 +0000 Subject: [PATCH 3/5] Remove unused function and related test --- spec/helpers_spec.cr | 6 ------ src/invidious/channels.cr | 25 ------------------------- 2 files changed, 31 deletions(-) diff --git a/spec/helpers_spec.cr b/spec/helpers_spec.cr index a4aaff9fe..a58c1e5aa 100644 --- a/spec/helpers_spec.cr +++ b/spec/helpers_spec.cr @@ -41,12 +41,6 @@ describe "Helper" do end end - describe "#extract_channel_playlists_cursor" do - it "correctly extracts a playlists cursor from the given URL" do - extract_channel_playlists_cursor("4qmFsgLRARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrQBRWdsd2JHRjViR2x6ZEhNWUF5QUJNQUk0QVdBQmFnQjZabEZWYkZCaE1XczFVbFpHZDJGV09XNWxWelI0V0RGR2VWSnVWbUZOV0Vwc1ZHcG5lRmd3TVU1aVZXdDRWMWN4YzFGdFNuTmtlbWh4VGpCd1NWTllVa1pTYTJNeFlVUmtlRmt3Y0ZWVWJWRXdWbnBzTkU1V1JqRmhNVGxFVm14dmQwMXFhRzVXZDdnQkFBJTNEJTNE", false).should eq("AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW") - end - end - describe "#produce_playlist_continuation" do it "correctly produces ctoken for requesting index `x` of a playlist" do produce_playlist_continuation("UUCla9fZca4I7KagBtgRGnOw", 100).should eq("4qmFsgJNEhpWTFVVQ2xhOWZaY2E0STdLYWdCdGdSR25PdxoUQ0FGNkJsQlVPa05IVVElM0QlM0SaAhhVVUNsYTlmWmNhNEk3S2FnQnRnUkduT3c%3D") diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 3a9b86410..30138d82f 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -512,31 +512,6 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end -def extract_channel_playlists_cursor(cursor, auto_generated) - cursor = URI.decode_www_form(cursor) - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h.find { |k, v| k.starts_with? "15:" } } - .try &.[1] - - if cursor.try &.as_h? - cursor = cursor.try { |i| Protodec::Any.cast_json(i.as_h) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } || "" - else - cursor = cursor.try &.as_s || "" - end - - if !auto_generated - cursor = URI.decode_www_form(cursor) - .try { |i| Base64.decode_string(i) } - end - - return cursor -end - # TODO: Add "sort_by" def fetch_channel_community(ucid, continuation, locale, format, thin_mode) response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") From c5ccefe6f7c127625c821d3b6836c23962a2b5b0 Mon Sep 17 00:00:00 2001 From: Svallinn <41585298+Svallinn@users.noreply.github.com> Date: Fri, 26 Mar 2021 03:52:28 +0000 Subject: [PATCH 4/5] Parse response to JSON instead of using regex --- src/invidious/channels.cr | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 30138d82f..2d5b44751 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -358,16 +358,20 @@ end def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation response_json = request_youtube_api_browse(continuation) - result = JSON.parse(response_json.match(/"continuationItems": (?\[.*\]),/m).try &.["items"] || "{}") + # result = JSON.parse(response_json.match(/"continuationItems": (?\[.*\]),/m).try &.["items"] || "{}") + result = JSON.parse(response_json) + continuationItems = result["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - return [] of SearchItem, nil if result.size == 0 + return [] of SearchItem, nil if !continuationItems items = [] of SearchItem - result.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| + continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| extract_item(item, author, ucid).try { |t| items << t } } - continuation = result.as_a.last["continuationItemRenderer"]?.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s + continuation = continuationItems.as_a.last["continuationItemRenderer"]? + .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s else url = "/channel/#{ucid}/playlists?flow=list&view=1" From 8823753b4680e5e0589f3be4d1d2b0cbc4344dc9 Mon Sep 17 00:00:00 2001 From: Svallinn <41585298+Svallinn@users.noreply.github.com> Date: Fri, 26 Mar 2021 03:54:10 +0000 Subject: [PATCH 5/5] Remove commented line --- src/invidious/channels.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 2d5b44751..47dfcbd6f 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -358,7 +358,6 @@ end def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation response_json = request_youtube_api_browse(continuation) - # result = JSON.parse(response_json.match(/"continuationItems": (?\[.*\]),/m).try &.["items"] || "{}") result = JSON.parse(response_json) continuationItems = result["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"]