Merge branch 'master' of https://github.com/iv-org/invidious

The function names `count_users_active_6m` and `count_users_active_1m` were
swapped. As the names were swapped on both sides (declaration and use), this
had no functional impact.

No related isse was tracked.

assets/css/default.css

@@ -197,6 +197,7 @@ img.thumbnail {
   display: block; /* See: https://stackoverflow.com/a/11635197 */
   width: 100%;
   object-fit: cover;
+  aspect-ratio: 16 / 9;
 }
 
 .thumbnail-placeholder {

assets/js/notifications.js

@@ -10,7 +10,7 @@ var notifications, delivered;
 var notifications_mock = { close: function () { } };
 
 function get_subscriptions() {
-    helpers.xhr('GET', '/api/v1/auth/subscriptions?fields=authorId', {
+    helpers.xhr('GET', '/api/v1/auth/subscriptions', {
         retries: 5,
         entity_name: 'subscriptions'
     }, {
@@ -22,7 +22,7 @@ function create_notification_stream(subscriptions) {
     // sse.js can't be replaced to EventSource in place as it lack support of payload and headers
     // see https://developer.mozilla.org/en-US/docs/Web/API/EventSource/EventSource
     notifications = new SSE(
-        '/api/v1/auth/notifications?fields=videoId,title,author,authorId,publishedText,published,authorThumbnails,liveNow', {
+        '/api/v1/auth/notifications', {
             withCredentials: true,
             payload: 'topics=' + subscriptions.map(function (subscription) { return subscription.authorId; }).join(','),
             headers: { 'Content-Type': 'application/x-www-form-urlencoded' }

locales/en-US.json

@@ -487,5 +487,6 @@
     "channel_tab_releases_label": "Releases",
     "channel_tab_playlists_label": "Playlists",
     "channel_tab_community_label": "Community",
-    "channel_tab_channels_label": "Channels"
+    "channel_tab_channels_label": "Channels",
+    "toggle_theme": "Toggle Theme"
 }

spec/invidious/search/yt_filters_spec.cr

@@ -12,45 +12,45 @@ end
 # page of Youtube with any browser devtools HTML inspector.
 
 DATE_FILTERS = {
-  Invidious::Search::Filters::Date::Hour  => "EgIIAQ%3D%3D",
-  Invidious::Search::Filters::Date::Today => "EgIIAg%3D%3D",
-  Invidious::Search::Filters::Date::Week  => "EgIIAw%3D%3D",
-  Invidious::Search::Filters::Date::Month => "EgIIBA%3D%3D",
-  Invidious::Search::Filters::Date::Year  => "EgIIBQ%3D%3D",
+  Invidious::Search::Filters::Date::Hour  => "EgIIAfABAQ%3D%3D",
+  Invidious::Search::Filters::Date::Today => "EgIIAvABAQ%3D%3D",
+  Invidious::Search::Filters::Date::Week  => "EgIIA_ABAQ%3D%3D",
+  Invidious::Search::Filters::Date::Month => "EgIIBPABAQ%3D%3D",
+  Invidious::Search::Filters::Date::Year  => "EgIIBfABAQ%3D%3D",
 }
 
 TYPE_FILTERS = {
-  Invidious::Search::Filters::Type::Video    => "EgIQAQ%3D%3D",
-  Invidious::Search::Filters::Type::Channel  => "EgIQAg%3D%3D",
-  Invidious::Search::Filters::Type::Playlist => "EgIQAw%3D%3D",
-  Invidious::Search::Filters::Type::Movie    => "EgIQBA%3D%3D",
+  Invidious::Search::Filters::Type::Video    => "EgIQAfABAQ%3D%3D",
+  Invidious::Search::Filters::Type::Channel  => "EgIQAvABAQ%3D%3D",
+  Invidious::Search::Filters::Type::Playlist => "EgIQA_ABAQ%3D%3D",
+  Invidious::Search::Filters::Type::Movie    => "EgIQBPABAQ%3D%3D",
 }
 
 DURATION_FILTERS = {
-  Invidious::Search::Filters::Duration::Short  => "EgIYAQ%3D%3D",
-  Invidious::Search::Filters::Duration::Medium => "EgIYAw%3D%3D",
-  Invidious::Search::Filters::Duration::Long   => "EgIYAg%3D%3D",
+  Invidious::Search::Filters::Duration::Short  => "EgIYAfABAQ%3D%3D",
+  Invidious::Search::Filters::Duration::Medium => "EgIYA_ABAQ%3D%3D",
+  Invidious::Search::Filters::Duration::Long   => "EgIYAvABAQ%3D%3D",
 }
 
 FEATURE_FILTERS = {
-  Invidious::Search::Filters::Features::Live       => "EgJAAQ%3D%3D",
-  Invidious::Search::Filters::Features::FourK      => "EgJwAQ%3D%3D",
-  Invidious::Search::Filters::Features::HD         => "EgIgAQ%3D%3D",
-  Invidious::Search::Filters::Features::Subtitles  => "EgIoAQ%3D%3D",
-  Invidious::Search::Filters::Features::CCommons   => "EgIwAQ%3D%3D",
-  Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AQ%3D%3D",
-  Invidious::Search::Filters::Features::VR180      => "EgPQAQE%3D",
-  Invidious::Search::Filters::Features::ThreeD     => "EgI4AQ%3D%3D",
-  Invidious::Search::Filters::Features::HDR        => "EgPIAQE%3D",
-  Invidious::Search::Filters::Features::Location   => "EgO4AQE%3D",
-  Invidious::Search::Filters::Features::Purchased  => "EgJIAQ%3D%3D",
+  Invidious::Search::Filters::Features::Live       => "EgJAAfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::FourK      => "EgJwAfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::HD         => "EgIgAfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::Subtitles  => "EgIoAfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::CCommons   => "EgIwAfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::VR180      => "EgPQAQHwAQE%3D",
+  Invidious::Search::Filters::Features::ThreeD     => "EgI4AfABAQ%3D%3D",
+  Invidious::Search::Filters::Features::HDR        => "EgPIAQHwAQE%3D",
+  Invidious::Search::Filters::Features::Location   => "EgO4AQHwAQE%3D",
+  Invidious::Search::Filters::Features::Purchased  => "EgJIAfABAQ%3D%3D",
 }
 
 SORT_FILTERS = {
-  Invidious::Search::Filters::Sort::Relevance => "",
-  Invidious::Search::Filters::Sort::Date      => "CAI%3D",
-  Invidious::Search::Filters::Sort::Views     => "CAM%3D",
-  Invidious::Search::Filters::Sort::Rating    => "CAE%3D",
+  Invidious::Search::Filters::Sort::Relevance => "8AEB",
+  Invidious::Search::Filters::Sort::Date      => "CALwAQE%3D",
+  Invidious::Search::Filters::Sort::Views     => "CAPwAQE%3D",
+  Invidious::Search::Filters::Sort::Rating    => "CAHwAQE%3D",
 }
 
 Spectator.describe Invidious::Search::Filters do

src/invidious.cr

@@ -217,7 +217,6 @@ public_folder "assets"
 
 Kemal.config.powered_by_header = false
 add_handler FilteredCompressHandler.new
-add_handler APIHandler.new
 add_handler AuthHandler.new
 add_handler DenyFrame.new
 add_context_storage_type(Array(String))

src/invidious/database/statistics.cr

@@ -15,7 +15,7 @@ module Invidious::Database::Statistics
     PG_DB.query_one(request, as: Int64)
   end
 
-  def count_users_active_1m : Int64
+  def count_users_active_6m : Int64
     request = <<-SQL
       SELECT count(*) FROM users
       WHERE CURRENT_TIMESTAMP - updated < '6 months'
@@ -24,7 +24,7 @@ module Invidious::Database::Statistics
     PG_DB.query_one(request, as: Int64)
   end
 
-  def count_users_active_6m : Int64
+  def count_users_active_1m : Int64
     request = <<-SQL
       SELECT count(*) FROM users
       WHERE CURRENT_TIMESTAMP - updated < '1 month'

src/invidious/helpers/handlers.cr

@@ -134,74 +134,6 @@ class AuthHandler < Kemal::Handler
   end
 end
 
-class APIHandler < Kemal::Handler
-  {% for method in %w(GET POST PUT HEAD DELETE PATCH OPTIONS) %}
-  only ["/api/v1/*"], {{method}}
-  {% end %}
-  exclude ["/api/v1/auth/notifications"], "GET"
-  exclude ["/api/v1/auth/notifications"], "POST"
-
-  def call(env)
-    return call_next env unless only_match? env
-
-    env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-    # Since /api/v1/notifications is an event-stream, we don't want
-    # to wrap the response
-    return call_next env if exclude_match? env
-
-    # Here we swap out the socket IO so we can modify the response as needed
-    output = env.response.output
-    env.response.output = IO::Memory.new
-
-    begin
-      call_next env
-
-      env.response.output.rewind
-
-      if env.response.output.as(IO::Memory).size != 0 &&
-         env.response.headers.includes_word?("Content-Type", "application/json")
-        response = JSON.parse(env.response.output)
-
-        if fields_text = env.params.query["fields"]?
-          begin
-            JSONFilter.filter(response, fields_text)
-          rescue ex
-            env.response.status_code = 400
-            response = {"error" => ex.message}
-          end
-        end
-
-        if env.params.query["pretty"]?.try &.== "1"
-          response = response.to_pretty_json
-        else
-          response = response.to_json
-        end
-      else
-        response = env.response.output.gets_to_end
-      end
-    rescue ex
-      env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html")
-      env.response.status_code = 500
-
-      if env.response.headers.includes_word?("Content-Type", "application/json")
-        response = {"error" => ex.message || "Unspecified error"}
-
-        if env.params.query["pretty"]?.try &.== "1"
-          response = response.to_pretty_json
-        else
-          response = response.to_json
-        end
-      end
-    ensure
-      env.response.output = output
-      env.response.print response
-
-      env.response.flush
-    end
-  end
-end
-
 class DenyFrame < Kemal::Handler
   exclude ["/embed/*"]
 

src/invidious/helpers/helpers.cr

@@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel)
           video.published = published
           response = JSON.parse(video.to_json(locale, nil))
 
-          if fields_text = env.params.query["fields"]?
-            begin
-              JSONFilter.filter(response, fields_text)
-            rescue ex
-              env.response.status_code = 400
-              response = {"error" => ex.message}
-            end
-          end
-
           env.response.puts "id: #{id}"
           env.response.puts "data: #{response.to_json}"
           env.response.puts
@@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel)
             Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
               response = JSON.parse(video.to_json(locale))
 
-              if fields_text = env.params.query["fields"]?
-                begin
-                  JSONFilter.filter(response, fields_text)
-                rescue ex
-                  env.response.status_code = 400
-                  response = {"error" => ex.message}
-                end
-              end
-
               env.response.puts "id: #{id}"
               env.response.puts "data: #{response.to_json}"
               env.response.puts
@@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel)
         video.published = Time.unix(published)
         response = JSON.parse(video.to_json(locale, nil))
 
-        if fields_text = env.params.query["fields"]?
-          begin
-            JSONFilter.filter(response, fields_text)
-          rescue ex
-            env.response.status_code = 400
-            response = {"error" => ex.message}
-          end
-        end
-
         env.response.puts "id: #{id}"
         env.response.puts "data: #{response.to_json}"
         env.response.puts

src/invidious/helpers/json_filter.cr

@@ -1,248 +0,0 @@
-module JSONFilter
-  alias BracketIndex = Hash(Int64, Int64)
-
-  alias GroupedFieldsValue = String | Array(GroupedFieldsValue)
-  alias GroupedFieldsList = Array(GroupedFieldsValue)
-
-  class FieldsParser
-    class ParseError < Exception
-    end
-
-    # Returns the `Regex` pattern used to match nest groups
-    def self.nest_group_pattern : Regex
-      # uses a '.' character to match json keys as they are allowed
-      # to contain any unicode codepoint
-      /(?:|,)(?<groupname>[^,\n]*?)\(/
-    end
-
-    # Returns the `Regex` pattern used to check if there are any empty nest groups
-    def self.unnamed_nest_group_pattern : Regex
-      /^\(|\(\(|\/\(/
-    end
-
-    def self.parse_fields(fields_text : String, &) : Nil
-      if fields_text.empty?
-        raise FieldsParser::ParseError.new "Fields is empty"
-      end
-
-      opening_bracket_count = fields_text.count('(')
-      closing_bracket_count = fields_text.count(')')
-
-      if opening_bracket_count != closing_bracket_count
-        bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing"
-        raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})"
-      elsif match_result = unnamed_nest_group_pattern.match(fields_text)
-        raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}"
-      end
-
-      # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc
-      parse_single_nests(fields_text) { |nest_list| yield nest_list }
-
-      # next, handle nest groups: items(id, etag, etc)
-      parse_nest_groups(fields_text) { |nest_list| yield nest_list }
-    end
-
-    def self.parse_single_nests(fields_text : String, &) : Nil
-      single_nests = remove_nest_groups(fields_text)
-
-      if !single_nests.empty?
-        property_nests = single_nests.split(',')
-
-        property_nests.each do |nest|
-          nest_list = nest.split('/')
-          if nest_list.includes? ""
-            raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}"
-          end
-          yield nest_list
-        end
-        # else
-        #   raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}"
-      end
-    end
-
-    def self.parse_nest_groups(fields_text : String, &) : Nil
-      nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
-      bracket_pairs = get_bracket_pairs(fields_text, true)
-
-      text_index = 0
-      regex_index = 0
-
-      while regex_result = self.nest_group_pattern.match(fields_text, regex_index)
-        raw_match = regex_result[0]
-        group_name = regex_result["groupname"]
-
-        text_index = regex_result.begin
-        regex_index = regex_result.end
-
-        if text_index.nil? || regex_index.nil?
-          raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}"
-        end
-
-        offset = raw_match.starts_with?(',') ? 1 : 0
-
-        opening_bracket_index = (text_index + group_name.size) + offset
-        closing_bracket_index = bracket_pairs[opening_bracket_index]
-        content_start = opening_bracket_index + 1
-
-        content = fields_text[content_start...closing_bracket_index]
-
-        if content.empty?
-          raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}"
-        else
-          content = remove_nest_groups(content)
-        end
-
-        while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index]
-          if nest_stack.size
-            nest_stack.pop
-          end
-        end
-
-        group_name.split('/').each do |name|
-          nest_stack.push({
-            group_name:            name,
-            closing_bracket_index: closing_bracket_index,
-          })
-        end
-
-        if !content.empty?
-          properties = content.split(',')
-
-          properties.each do |prop|
-            nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] }
-
-            if !prop.empty?
-              if prop.includes?('/')
-                parse_single_nests(prop) { |list| nest_list += list }
-              else
-                nest_list.push prop
-              end
-            else
-              raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}"
-            end
-
-            yield nest_list
-          end
-        end
-      end
-    end
-
-    def self.remove_nest_groups(text : String) : String
-      content_bracket_pairs = get_bracket_pairs(text, false)
-
-      content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket|
-        closing_bracket = content_bracket_pairs[opening_bracket]
-        last_comma = text.rindex(',', opening_bracket) || 0
-
-        text = text[0...last_comma] + text[closing_bracket + 1...text.size]
-      end
-
-      return text.starts_with?(',') ? text[1...text.size] : text
-    end
-
-    def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex
-      istart = [] of Int64
-      bracket_index = BracketIndex.new
-
-      text.each_char_with_index do |char, index|
-        if char == '('
-          istart.push(index.to_i64)
-        end
-
-        if char == ')'
-          begin
-            opening = istart.pop
-            if recursive || (!recursive && istart.size == 0)
-              bracket_index[opening] = index.to_i64
-            end
-          rescue
-            raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}"
-          end
-        end
-      end
-
-      if istart.size != 0
-        idx = istart.pop
-        raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}"
-      end
-
-      return bracket_index
-    end
-  end
-
-  class FieldsGrouper
-    alias SkeletonValue = Hash(String, SkeletonValue)
-
-    def self.create_json_skeleton(fields_text : String) : SkeletonValue
-      root_hash = {} of String => SkeletonValue
-
-      FieldsParser.parse_fields(fields_text) do |nest_list|
-        current_item = root_hash
-        nest_list.each do |key|
-          if current_item[key]?
-            current_item = current_item[key]
-          else
-            current_item[key] = {} of String => SkeletonValue
-            current_item = current_item[key]
-          end
-        end
-      end
-      root_hash
-    end
-
-    def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList
-      grouped_fields_list = GroupedFieldsList.new
-      json_skeleton.each do |key, value|
-        grouped_fields_list.push key
-
-        nested_keys = create_grouped_fields_list(value)
-        grouped_fields_list.push nested_keys unless nested_keys.empty?
-      end
-      return grouped_fields_list
-    end
-  end
-
-  class FilterError < Exception
-  end
-
-  def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true)
-    skeleton = FieldsGrouper.create_json_skeleton(fields_text)
-    grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton)
-    filter(item, grouped_fields_list, in_place)
-  end
-
-  def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any
-    item = item.clone unless in_place
-
-    if !item.as_h? && !item.as_a?
-      raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}"
-    end
-
-    top_level_keys = Array(String).new
-    grouped_fields_list.each do |value|
-      if value.is_a? String
-        top_level_keys.push value
-      elsif value.is_a? Array
-        if !top_level_keys.empty?
-          key_to_filter = top_level_keys.last
-
-          if item.as_h?
-            filter(item[key_to_filter], value, in_place: true)
-          elsif item.as_a?
-            item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) }
-          end
-        else
-          raise FilterError.new "Tried to filter while top level keys list is empty"
-        end
-      end
-    end
-
-    if item.as_h?
-      item.as_h.select! top_level_keys
-    elsif item.as_a?
-      item.as_a.map { |value| filter(value, top_level_keys, in_place: true) }
-    end
-
-    item
-  end
-end

src/invidious/helpers/utils.cr

@@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true)
   end
 
   referer = referer.request_target
-  referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\")
+  referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\")
 
   if referer == env.request.path
     referer = fallback

src/invidious/jobs/statistics_refresh_job.cr

@@ -56,8 +56,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
     users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
 
     users["total"] = Invidious::Database::Statistics.count_users_total
-    users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m
-    users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m
+    users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m
+    users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m
 
     STATISTICS["metadata"] = {
       "updatedAt"              => Time.utc.to_unix,

src/invidious/routes/api/v1/misc.cr

@@ -191,6 +191,8 @@ module Invidious::Routes::API::V1::Misc
       json.object do
         json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
         json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
+        json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
+        json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
         json.field "params", params.try &.as_s
         json.field "pageType", pageType
       end

src/invidious/routes/api/v1/search.cr

@@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search
 
     begin
       client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
-      url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt"
+      client.before_request { |r| add_yt_headers(r) }
+
+      url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
 
       response = client.get(url).body
+      client.close
 
-      body = JSON.parse(response[5..-1]).as_a
+      body = JSON.parse(response[19..-2]).as_a
       suggestions = body[1].as_a[0..-2]
 
       JSON.build do |json|

src/invidious/routes/api/v1/videos.cr

@@ -363,4 +363,47 @@ module Invidious::Routes::API::V1::Videos
       end
     end
   end
+
+  def self.clips(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    env.response.content_type = "application/json"
+
+    clip_id = env.params.url["id"]
+    region = env.params.query["region"]?
+    proxy = {"1", "true"}.any? &.== env.params.query["local"]?
+
+    response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
+    return error_json(400, "Invalid clip ID") if response["error"]?
+
+    video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
+    return error_json(400, "Invalid clip ID") if video_id.nil?
+
+    start_time = nil
+    end_time = nil
+    clip_title = nil
+
+    if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
+      start_time, end_time, clip_title = parse_clip_parameters(params)
+    end
+
+    begin
+      video = get_video(video_id, region: region)
+    rescue ex : NotFoundException
+      return error_json(404, ex)
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    return JSON.build do |json|
+      json.object do
+        json.field "startTime", start_time
+        json.field "endTime", end_time
+        json.field "clipTitle", clip_title
+        json.field "video" do
+          Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
+        end
+      end
+    end
+  end
 end

src/invidious/routes/feeds.cr

@@ -407,14 +407,23 @@ module Invidious::Routes::Feeds
     end
 
     spawn do
-      rss = XML.parse_html(body)
-      rss.xpath_nodes("//feed/entry").each do |entry|
-        id = entry.xpath_node("videoid").not_nil!.content
-        author = entry.xpath_node("author/name").not_nil!.content
-        published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
-        updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+      # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
+      namespaces = {
+        "yt"      => "http://www.youtube.com/xml/schemas/2015",
+        "default" => "http://www.w3.org/2005/Atom",
+      }
+      rss = XML.parse(body)
+      rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
+        id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+        author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+        published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
+        updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
 
-        video = get_video(id, force_refresh: true)
+        begin
+          video = get_video(id, force_refresh: true)
+        rescue
+          next # skip this video since it raised an exception (e.g. it is a scheduled live event)
+        end
 
         if CONFIG.enable_user_notifications
           # Deliver notifications to `/api/v1/auth/notifications`

src/invidious/routes/watch.cr

@@ -275,6 +275,12 @@ module Invidious::Routes::Watch
     return error_template(400, "Invalid clip ID") if response["error"]?
 
     if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
+      if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
+        start_time, end_time, _ = parse_clip_parameters(params)
+        env.params.query["start"] = start_time.to_s if start_time != nil
+        env.params.query["end"] = end_time.to_s if end_time != nil
+      end
+
       return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
     else
       return error_template(404, "The requested clip doesn't exist")

src/invidious/routing.cr

@@ -234,6 +234,7 @@ module Invidious::Routing
       get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
       get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
       get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
+      get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
 
       # Feeds
       get "/api/v1/trending", {{namespace}}::Feeds, :trending

src/invidious/search/filters.cr

@@ -300,9 +300,9 @@ module Invidious::Search
         object["9:varint"] = ((page - 1) * 20).to_i64
       end
 
-      # If the object is empty, return an empty string,
-      # otherwise encode to protobuf then to base64
-      return "" if object.empty?
+      # Prevent censoring of self harm topics
+      # See https://github.com/iv-org/invidious/issues/4398
+      object["30:varint"] = 1.to_i64
 
       return object
         .try { |i| Protodec::Any.cast_json(i) }

src/invidious/videos/clip.cr

@@ -0,0 +1,22 @@
+require "json"
+
+# returns start_time, end_time and clip_title
+def parse_clip_parameters(params) : {Float64?, Float64?, String?}
+  decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
+    .try { |i| Base64.decode(i) }
+    .try { |i| IO::Memory.new(i) }
+    .try { |i| Protodec::Any.parse(i) }
+
+  start_time = decoded_protobuf
+    .try(&.["50:0:embedded"]["2:1:varint"].as_i64)
+    .try { |i| i/1000 }
+
+  end_time = decoded_protobuf
+    .try(&.["50:0:embedded"]["3:2:varint"].as_i64)
+    .try { |i| i/1000 }
+
+  clip_title = decoded_protobuf
+    .try(&.["50:0:embedded"]["4:3:string"].as_s)
+
+  return start_time, end_time, clip_title
+end

src/invidious/views/components/item.ecr

@@ -82,11 +82,19 @@
             </div>
 
             <div class="video-card-row flexible">
-                <div class="flex-left"><a href="/channel/<%= item.ucid %>">
-                    <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
-                        <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
-                    </p>
-                </a></div>
+                <div class="flex-left">
+                    <% if !item.ucid.to_s.empty? %>
+                        <a href="/channel/<%= item.ucid %>">
+                            <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+                                <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+                            </p>
+                        </a>
+                    <% else %>
+                        <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+                            <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+                        </p>
+                    <% end %>
+                </div>
             </div>
         <% when Category %>
         <% else %>
@@ -160,11 +168,19 @@
             </div>
 
             <div class="video-card-row flexible">
-                <div class="flex-left"><a href="/channel/<%= item.ucid %>">
-                    <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
-                        <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
-                    </p>
-                </a></div>
+                <div class="flex-left">
+                    <% if !item.ucid.to_s.empty? %>
+                        <a href="/channel/<%= item.ucid %>">
+                            <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+                                <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+                            </p>
+                        </a>
+                    <% else %>
+                        <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+                            <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+                        </p>
+                    <% end %>
+                </div>
 
                 <%= rendered "components/video-context-buttons" %>
             </div>

src/invidious/views/template.ecr

@@ -1,5 +1,9 @@
+<%
+  locale = env.get("preferences").as(Preferences).locale
+  dark_mode = env.get("preferences").as(Preferences).dark_mode
+%>
 <!DOCTYPE html>
-<html lang="<%= env.get("preferences").as(Preferences).locale %>">
+<html lang="<%= locale %>">
 
 <head>
     <meta charset="utf-8">

src/invidious/views/watch.ecr

@@ -118,7 +118,7 @@ we're going to need to do it here in order to allow for translations.
                     link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}")
 
                     if !plid.nil? && !continuation.nil?
-                        link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]}
+                        link_yt_param = URI::Params{"list" => [plid], "index" => [continuation.to_s]}
                         link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param)
                         link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param)
                     end
@@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations.
 
                             <h5 class="pure-g">
                                 <div class="pure-u-14-24">
-                                    <% if rv["ucid"]? %>
+                                    <% if !rv["ucid"].empty? %>
                                         <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b>
                                     <% else %>
                                         <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %></b>

src/invidious/yt_backend/extractors.cr

@@ -822,9 +822,9 @@ module HelperExtractors
   end
 
   # Retrieves the ID required for querying the InnerTube browse endpoint.
-  # Raises when it's unable to do so
+  # Returns an empty string when it's unable to do so
   def self.get_browse_id(container)
-    return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s
+    return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
   end
 end