From d93a7b315db42474aac4a8e27c3745dc4b5abdeb Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 26 Mar 2022 20:15:02 +0100 Subject: [PATCH] Make use of Search::Query/Filters and associated HTML generator --- src/invidious/routes/api/v1/channels.cr | 16 ++- src/invidious/routes/api/v1/search.cr | 24 +--- src/invidious/routes/playlists.cr | 18 ++- src/invidious/routes/search.cr | 24 ++-- src/invidious/search.cr | 150 +-------------------- src/invidious/search/filters.cr | 2 +- src/invidious/search/processors.cr | 28 ++-- src/invidious/search/query.cr | 5 +- src/invidious/views/add_playlist_items.ecr | 11 +- src/invidious/views/search.ecr | 140 ++++--------------- 10 files changed, 87 insertions(+), 331 deletions(-) diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4395353..8650976d 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels def self.search(env) locale = env.get("preferences").as(Preferences).locale + region = env.params.query["region"]? env.response.content_type = "application/json" - ucid = env.params.url["ucid"] + query = Invidious::Search::Query.new(env.params.query, :channel, region) - query = env.params.query["q"]? - query ||= "" + # Required because we can't (yet) pass multiple parameter to the + # `Search::Query` initializer (in this case, an URL segment) + query.channel = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + begin + search_results = query.process + rescue ex + return error_json(400, ex) + end - search_results = Invidious::Search::Processors.channel(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 5666460d..21451d33 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search env.response.content_type = "application/json" - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map(&.downcase) - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" + query = Invidious::Search::Query.new(env.params.query, :regular, region) begin - search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + search_results = query.process rescue ex return error_json(400, ex) end - search_results = search(query, search_params, region) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index dbeb4f97..de981d81 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -212,7 +212,10 @@ module Invidious::Routes::Playlists end def self.add_playlist_items_page(env) - locale = env.get("preferences").as(Preferences).locale + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale + + region = env.params.query["region"]? || prefs.region user = env.get? "user" sid = env.get? "sid" @@ -236,15 +239,10 @@ module Invidious::Routes::Playlists return env.redirect referer end - query = env.params.query["q"]? - if query - begin - search_query, items, operators = process_search_query(query, page, user, region: nil) - videos = items.select(SearchVideo).map(&.as(SearchVideo)) - rescue ex - videos = [] of SearchVideo - end - else + begin + query = Invidious::Search::Query.new(env.params.query, :playlist, region) + videos = query.process.select(SearchVideo).map(&.as(SearchVideo)) + rescue ex videos = [] of SearchVideo end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 3f4c7e5e..e60d0081 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -37,37 +37,29 @@ module Invidious::Routes::Search end def self.search(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? + region = env.params.query["region"]? || prefs.region - if !query || query.empty? + query = Invidious::Search::Query.new(env.params.query, :regular, region) + + if query.empty? # Display the full page search box implemented in #1977 env.set "search", "" templated "search_homepage", navbar_search: false else - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - user = env.get? "user" begin - search_query, videos, operators = process_search_query(query, page, user, region: region) + videos = query.process rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value - end - - env.set "search", query + env.set "search", query.text templated "search" end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index af854653..e4c21bd4 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -5,113 +5,6 @@ class ChannelSearchException < InfoException end end -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) - return [] of SearchItem if query.empty? - - client_config = YoutubeAPI::ClientConfig.new(region: region) - initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) - - return extract_items(initial_data) -end - -def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", - duration : String = "", features : Array(String) = [] of String) - object = { - "1:varint" => 0_i64, - "2:embedded" => {} of String => Int64, - "9:varint" => ((page - 1) * 20).to_i64, - } - - case sort - when "relevance" - object["1:varint"] = 0_i64 - when "rating" - object["1:varint"] = 1_i64 - when "upload_date", "date" - object["1:varint"] = 2_i64 - when "view_count", "views" - object["1:varint"] = 3_i64 - else - raise "No sort #{sort}" - end - - case date - when "hour" - object["2:embedded"].as(Hash)["1:varint"] = 1_i64 - when "today" - object["2:embedded"].as(Hash)["1:varint"] = 2_i64 - when "week" - object["2:embedded"].as(Hash)["1:varint"] = 3_i64 - when "month" - object["2:embedded"].as(Hash)["1:varint"] = 4_i64 - when "year" - object["2:embedded"].as(Hash)["1:varint"] = 5_i64 - else nil # Ignore - end - - case content_type - when "video" - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - when "channel" - object["2:embedded"].as(Hash)["2:varint"] = 2_i64 - when "playlist" - object["2:embedded"].as(Hash)["2:varint"] = 3_i64 - when "movie" - object["2:embedded"].as(Hash)["2:varint"] = 4_i64 - when "show" - object["2:embedded"].as(Hash)["2:varint"] = 5_i64 - when "all" - # - else - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - end - - case duration - when "short" - object["2:embedded"].as(Hash)["3:varint"] = 1_i64 - when "long" - object["2:embedded"].as(Hash)["3:varint"] = 2_i64 - else nil # Ignore - end - - features.each do |feature| - case feature - when "hd" - object["2:embedded"].as(Hash)["4:varint"] = 1_i64 - when "subtitles" - object["2:embedded"].as(Hash)["5:varint"] = 1_i64 - when "creative_commons", "cc" - object["2:embedded"].as(Hash)["6:varint"] = 1_i64 - when "3d" - object["2:embedded"].as(Hash)["7:varint"] = 1_i64 - when "live", "livestream" - object["2:embedded"].as(Hash)["8:varint"] = 1_i64 - when "purchased" - object["2:embedded"].as(Hash)["9:varint"] = 1_i64 - when "4k" - object["2:embedded"].as(Hash)["14:varint"] = 1_i64 - when "360" - object["2:embedded"].as(Hash)["15:varint"] = 1_i64 - when "location" - object["2:embedded"].as(Hash)["23:varint"] = 1_i64 - when "hdr" - object["2:embedded"].as(Hash)["25:varint"] = 1_i64 - else nil # Ignore - end - end - - if object["2:embedded"].as(Hash).empty? - object.delete("2:embedded") - end - - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return params -end - def produce_channel_search_continuation(ucid, query, page) if page <= 1 idx = 0_i64 @@ -146,41 +39,10 @@ def produce_channel_search_continuation(ucid, query, page) end def process_search_query(query, page, user, region) - channel = nil - content_type = "all" - date = "" - duration = "" - features = [] of String - sort = "relevance" - subscriptions = nil + # Parse legacy query + filters, channel, search_query, subscriptions = Invidious::Search::Filters.from_legacy_filters(query) - operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) - operators.each do |operator| - key, value = operator.downcase.split(":") - - case key - when "channel", "user" - channel = operator.split(":")[-1] - when "content_type", "type" - content_type = value - when "date" - date = value - when "duration" - duration = value - when "feature", "features" - features = value.split(",") - when "sort" - sort = value - when "subscriptions" - subscriptions = value == "true" - else - operators.delete(operator) - end - end - - search_query = (query.split(" ") - operators).join(" ") - - if channel + if !channel.nil? && !channel.empty? items = Invidious::Search::Processors.channel(search_query, page, channel) elsif subscriptions if user @@ -190,9 +52,7 @@ def process_search_query(query, page, user, region) items = [] of ChannelVideo end else - search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, - duration: duration, features: features) - + search_params = filters.to_yt_params(page: page) items = search(search_query, search_params, region) end @@ -211,5 +71,5 @@ def process_search_query(query, page, user, region) end end - {search_query, items_without_category, operators} + {search_query, items_without_category, filters} end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index 0e8438b9..c2b5c758 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -79,7 +79,7 @@ module Invidious::Search ) end - def is_default? : Bool + def default? : Bool return @date.none? && @type.all? && @duration.none? && \ @features.none? && @sort.relevance? end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index c5327f34..d1409c06 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -2,22 +2,32 @@ module Invidious::Search module Processors extend self + # Regular search (`/search` endpoint) + def regular(query : Query) : Array(SearchItem) + search_params = query.filters.to_yt_params(page: query.page) + + client_config = YoutubeAPI::ClientConfig.new(region: query.region) + initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) + + return extract_items(initial_data) + end + # Search a youtube channel # TODO: clean code, and rely more on YoutubeAPI - def channel(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") + def channel(query : Query) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{query.channel}") if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{query.channel}") + response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 initial_data = extract_initial_data(response.body) ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid + raise ChannelSearchException.new(query.channel) if !ucid else - ucid = channel + ucid = query.channel end - continuation = produce_channel_search_continuation(ucid, query, page) + continuation = produce_channel_search_continuation(ucid, query.text, query.page) response_json = YoutubeAPI.browse(continuation) continuation_items = response_json["onResponseReceivedActions"]? @@ -34,7 +44,7 @@ module Invidious::Search end # Search inside of user subscriptions - def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo) + def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) view_name = "subscriptions_#{sha256(user.email)}" return PG_DB.query_all(" @@ -46,7 +56,7 @@ module Invidious::Search as document FROM #{view_name} ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", - query, (page - 1) * 20, + query.text, (query.page - 1) * 20, as: ChannelVideo ) end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index 4d76b083..1c2b37d2 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -110,11 +110,10 @@ module Invidious::Search case @type when .regular?, .playlist? - all_items = search(@query, @filters, @page, @region) - items = unnest_items(all_items) + items = unnest_items(Processors.regular(self)) # when .channel? - items = Processors.channel(@query, @page, @channel) + items = Processors.channel(self) # when .subscriptions? if user diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index ad50909a..22870317 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -11,7 +11,9 @@ <%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %>
- value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>> + value="<%= HTML.escape(query.text) %>"<% end %> + placeholder="<%= translate(locale, "Search for videos") %>">
@@ -38,10 +40,11 @@ <% if query %> + <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
- <% if page > 1 %> - + <% if query.page > 1 %> + <%= translate(locale, "Previous page") %> <% end %> @@ -49,7 +52,7 @@
<% if videos.size >= 20 %> - + <%= translate(locale, "Next page") %> <% end %> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 45bbdefc..f1f6ab20 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -1,124 +1,38 @@ <% content_for "header" do %> -<%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious +<%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "…" : HTML.escape(query.text) %> - Invidious + <% end %> -<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> +<%- + search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true) + filter_params = query.filters.to_iv_params + + url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}" + url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}" +-%> <% if videos.size == 0 %>

"><%= translate(locale, "Broken? Try another Invidious Instance!") %>

-<% else %> -
- -

<%= translate(locale, "filter") %>

- -
-
- <%= translate(locale, "date") %> -
- <% ["hour", "today", "week", "month", "year"].each do |date| %> -
- <% if operator_hash.fetch("date", "all") == date %> - <%= translate(locale, date) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, date) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "content_type") %> -
- <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %> -
- <% if operator_hash.fetch("content_type", "all") == content_type %> - <%= translate(locale, content_type) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, content_type) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "duration") %> -
- <% ["short", "long"].each do |duration| %> -
- <% if operator_hash.fetch("duration", "all") == duration %> - <%= translate(locale, duration) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, duration) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "features") %> -
- <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %> -
- <% if operator_hash.fetch("features", "all").includes?(feature) %> - <%= translate(locale, feature) %> - <% elsif operator_hash.has_key?("features") %> - &page=<%= page %>"> - <%= translate(locale, feature) %> - - <% else %> - &page=<%= page %>"> - <%= translate(locale, feature) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "sort") %> -
- <% ["relevance", "rating", "date", "views"].each do |sort| %> -
- <% if operator_hash.fetch("sort", "relevance") == sort %> - <%= translate(locale, sort) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, sort) %> - - <% end %> -
- <% end %> -
-
-
-<% end %> +<%- else -%> + <%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %> +<%- end -%> -<% if videos.size == 0 %> -
-<% else %> -
-<% end %> +<% if videos.size == 0 %>
<% else %>
<% end %>
- <% if page > 1 %> - - <%= translate(locale, "Previous page") %> - - <% end %> + <%- if query.page > 1 -%> + <%= translate(locale, "Previous page") %> + <%- end -%>
- <% if videos.size >= 20 %> - - <%= translate(locale, "Next page") %> - - <% end %> + <%- if videos.size >= 20 -%> + <%= translate(locale, "Next page") %> + <%- end -%>
@@ -130,18 +44,14 @@
- <% if page > 1 %> - - <%= translate(locale, "Previous page") %> - - <% end %> + <%- if query.page > 1 -%> + <%= translate(locale, "Previous page") %> + <%- end -%>
- <% if videos.size >= 20 %> - - <%= translate(locale, "Next page") %> - - <% end %> + <%- if videos.size >= 20 -%> + <%= translate(locale, "Next page") %> + <%- end -%>