427 行
12 KiB
Crystal
427 行
12 KiB
Crystal
enum VideoType
|
|
Video
|
|
Livestream
|
|
Scheduled
|
|
end
|
|
|
|
struct Video
|
|
include DB::Serializable
|
|
|
|
# Version of the JSON structure
|
|
# It prevents us from loading an incompatible version from cache
|
|
# (either newer or older, if instances with different versions run
|
|
# concurrently, e.g during a version upgrade rollout).
|
|
#
|
|
# NOTE: don't forget to bump this number if any change is made to
|
|
# the `params` structure in videos/parser.cr!!!
|
|
#
|
|
SCHEMA_VERSION = 2
|
|
|
|
property id : String
|
|
|
|
@[DB::Field(converter: Video::JSONConverter)]
|
|
property info : Hash(String, JSON::Any)
|
|
property updated : Time
|
|
|
|
@[DB::Field(ignore: true)]
|
|
@captions = [] of Invidious::Videos::Caption
|
|
|
|
@[DB::Field(ignore: true)]
|
|
property adaptive_fmts : Array(Hash(String, JSON::Any))?
|
|
|
|
@[DB::Field(ignore: true)]
|
|
property fmt_stream : Array(Hash(String, JSON::Any))?
|
|
|
|
@[DB::Field(ignore: true)]
|
|
property description : String?
|
|
|
|
module JSONConverter
|
|
def self.from_rs(rs)
|
|
JSON.parse(rs.read(String)).as_h
|
|
end
|
|
end
|
|
|
|
# Methods for API v1 JSON
|
|
|
|
def to_json(locale : String?, json : JSON::Builder)
|
|
Invidious::JSONify::APIv1.video(self, json, locale: locale)
|
|
end
|
|
|
|
# TODO: remove the locale and follow the crystal convention
|
|
def to_json(locale : String?, _json : Nil)
|
|
JSON.build do |json|
|
|
Invidious::JSONify::APIv1.video(self, json, locale: locale)
|
|
end
|
|
end
|
|
|
|
def to_json(json : JSON::Builder | Nil = nil)
|
|
to_json(nil, json)
|
|
end
|
|
|
|
# Misc methods
|
|
|
|
def video_type : VideoType
|
|
video_type = info["videoType"]?.try &.as_s || "video"
|
|
return VideoType.parse?(video_type) || VideoType::Video
|
|
end
|
|
|
|
def schema_version : Int
|
|
return info["version"]?.try &.as_i || 1
|
|
end
|
|
|
|
def published : Time
|
|
return info["published"]?
|
|
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
|
|
end
|
|
|
|
def published=(other : Time)
|
|
info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
|
|
end
|
|
|
|
def live_now
|
|
return (self.video_type == VideoType::Livestream)
|
|
end
|
|
|
|
def premiere_timestamp : Time?
|
|
info
|
|
.dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp")
|
|
.try { |t| Time.parse_rfc3339(t.as_s) }
|
|
end
|
|
|
|
def related_videos
|
|
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
|
|
end
|
|
|
|
# Methods for parsing streaming data
|
|
|
|
def fmt_stream
|
|
return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
|
|
|
|
fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
|
|
fmt_stream.each do |fmt|
|
|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
|
|
s.each do |k, v|
|
|
fmt[k] = JSON::Any.new(v)
|
|
end
|
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
|
|
end
|
|
|
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
|
|
end
|
|
|
|
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
|
|
@fmt_stream = fmt_stream
|
|
return @fmt_stream.as(Array(Hash(String, JSON::Any)))
|
|
end
|
|
|
|
def adaptive_fmts
|
|
return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
|
|
fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
|
|
fmt_stream.each do |fmt|
|
|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
|
|
s.each do |k, v|
|
|
fmt[k] = JSON::Any.new(v)
|
|
end
|
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
|
|
end
|
|
|
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
|
|
end
|
|
|
|
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
|
|
@adaptive_fmts = fmt_stream
|
|
return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
|
|
end
|
|
|
|
def video_streams
|
|
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
|
|
end
|
|
|
|
def audio_streams
|
|
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
|
|
end
|
|
|
|
# Misc. methods
|
|
|
|
def storyboards
|
|
storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
|
|
.try &.as_s.split("|")
|
|
|
|
if !storyboards
|
|
if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
|
|
return [{
|
|
url: storyboard.split("#")[0],
|
|
width: 106,
|
|
height: 60,
|
|
count: -1,
|
|
interval: 5000,
|
|
storyboard_width: 3,
|
|
storyboard_height: 3,
|
|
storyboard_count: -1,
|
|
}]
|
|
end
|
|
end
|
|
|
|
items = [] of NamedTuple(
|
|
url: String,
|
|
width: Int32,
|
|
height: Int32,
|
|
count: Int32,
|
|
interval: Int32,
|
|
storyboard_width: Int32,
|
|
storyboard_height: Int32,
|
|
storyboard_count: Int32)
|
|
|
|
return items if !storyboards
|
|
|
|
url = URI.parse(storyboards.shift)
|
|
params = HTTP::Params.parse(url.query || "")
|
|
|
|
storyboards.each_with_index do |sb, i|
|
|
width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
|
|
params["sigh"] = sigh
|
|
url.query = params.to_s
|
|
|
|
width = width.to_i
|
|
height = height.to_i
|
|
count = count.to_i
|
|
interval = interval.to_i
|
|
storyboard_width = storyboard_width.to_i
|
|
storyboard_height = storyboard_height.to_i
|
|
storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
|
|
|
|
items << {
|
|
url: url.to_s.sub("$L", i).sub("$N", "M$M"),
|
|
width: width,
|
|
height: height,
|
|
count: count,
|
|
interval: interval,
|
|
storyboard_width: storyboard_width,
|
|
storyboard_height: storyboard_height,
|
|
storyboard_count: storyboard_count,
|
|
}
|
|
end
|
|
|
|
items
|
|
end
|
|
|
|
def paid
|
|
return (self.reason || "").includes? "requires payment"
|
|
end
|
|
|
|
def premium
|
|
keywords.includes? "YouTube Red"
|
|
end
|
|
|
|
def captions : Array(Invidious::Videos::Caption)
|
|
if @captions.empty? && @info.has_key?("captions")
|
|
@captions = Invidious::Videos::Caption.from_yt_json(info["captions"])
|
|
end
|
|
|
|
return @captions
|
|
end
|
|
|
|
def hls_manifest_url : String?
|
|
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
|
|
end
|
|
|
|
def dash_manifest_url
|
|
info.dig?("streamingData", "dashManifestUrl").try &.as_s
|
|
end
|
|
|
|
def genre_url : String?
|
|
info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
|
|
end
|
|
|
|
def is_vr : Bool?
|
|
return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
|
|
end
|
|
|
|
def projection_type : String?
|
|
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
|
|
end
|
|
|
|
def reason : String?
|
|
info["reason"]?.try &.as_s
|
|
end
|
|
|
|
# Macros defining getters/setters for various types of data
|
|
|
|
private macro getset_string(name)
|
|
# Return {{name.stringify}} from `info`
|
|
def {{name.id.underscore}} : String
|
|
return info[{{name.stringify}}]?.try &.as_s || ""
|
|
end
|
|
|
|
# Update {{name.stringify}} into `info`
|
|
def {{name.id.underscore}}=(value : String)
|
|
info[{{name.stringify}}] = JSON::Any.new(value)
|
|
end
|
|
|
|
{% if flag?(:debug_macros) %} {{debug}} {% end %}
|
|
end
|
|
|
|
private macro getset_string_array(name)
|
|
# Return {{name.stringify}} from `info`
|
|
def {{name.id.underscore}} : Array(String)
|
|
return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
|
|
end
|
|
|
|
# Update {{name.stringify}} into `info`
|
|
def {{name.id.underscore}}=(value : Array(String))
|
|
info[{{name.stringify}}] = JSON::Any.new(value)
|
|
end
|
|
|
|
{% if flag?(:debug_macros) %} {{debug}} {% end %}
|
|
end
|
|
|
|
{% for op, type in {i32: Int32, i64: Int64} %}
|
|
private macro getset_{{op}}(name)
|
|
def \{{name.id.underscore}} : {{type}}
|
|
return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}}
|
|
end
|
|
|
|
def \{{name.id.underscore}}=(value : Int)
|
|
info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
|
|
end
|
|
|
|
\{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
|
|
end
|
|
{% end %}
|
|
|
|
private macro getset_bool(name)
|
|
# Return {{name.stringify}} from `info`
|
|
def {{name.id.underscore}} : Bool
|
|
return info[{{name.stringify}}]?.try &.as_bool || false
|
|
end
|
|
|
|
# Update {{name.stringify}} into `info`
|
|
def {{name.id.underscore}}=(value : Bool)
|
|
info[{{name.stringify}}] = JSON::Any.new(value)
|
|
end
|
|
|
|
{% if flag?(:debug_macros) %} {{debug}} {% end %}
|
|
end
|
|
|
|
# Method definitions, using the macros above
|
|
|
|
getset_string author
|
|
getset_string authorThumbnail
|
|
getset_string description
|
|
getset_string descriptionHtml
|
|
getset_string genre
|
|
getset_string genreUcid
|
|
getset_string license
|
|
getset_string shortDescription
|
|
getset_string subCountText
|
|
getset_string title
|
|
getset_string ucid
|
|
|
|
getset_string_array allowedRegions
|
|
getset_string_array keywords
|
|
|
|
getset_i32 lengthSeconds
|
|
getset_i64 likes
|
|
getset_i64 views
|
|
|
|
getset_bool allowRatings
|
|
getset_bool authorVerified
|
|
getset_bool isFamilyFriendly
|
|
getset_bool isListed
|
|
getset_bool isUpcoming
|
|
end
|
|
|
|
def get_video(id, refresh = true, region = nil, force_refresh = false)
|
|
if (video = Invidious::Database::Videos.select(id)) && !region
|
|
# If record was last updated over 10 minutes ago, or video has since premiered,
|
|
# refresh (expire param in response lasts for 6 hours)
|
|
if (refresh &&
|
|
(Time.utc - video.updated > 10.minutes) ||
|
|
(video.premiere_timestamp.try &.< Time.utc)) ||
|
|
force_refresh ||
|
|
video.schema_version != Video::SCHEMA_VERSION # cache control
|
|
begin
|
|
video = fetch_video(id, region)
|
|
Invidious::Database::Videos.update(video)
|
|
rescue ex
|
|
Invidious::Database::Videos.delete(id)
|
|
raise ex
|
|
end
|
|
end
|
|
else
|
|
video = fetch_video(id, region)
|
|
Invidious::Database::Videos.insert(video) if !region
|
|
end
|
|
|
|
return video
|
|
rescue DB::Error
|
|
# Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
|
|
# Note: All DB errors inherit from `DB::Error`
|
|
return fetch_video(id, region)
|
|
end
|
|
|
|
def fetch_video(id, region)
|
|
info = extract_video_info(video_id: id)
|
|
|
|
allowed_regions = info
|
|
.dig?("microformat", "playerMicroformatRenderer", "availableCountries")
|
|
.try &.as_a.map &.as_s || [] of String
|
|
|
|
# Check for region-blocks
|
|
if info["reason"]?.try &.as_s.includes?("your country")
|
|
bypass_regions = PROXY_LIST.keys & allowed_regions
|
|
if !bypass_regions.empty?
|
|
region = bypass_regions[rand(bypass_regions.size)]
|
|
region_info = extract_video_info(video_id: id, proxy_region: region)
|
|
region_info["region"] = JSON::Any.new(region) if region
|
|
info = region_info if !region_info["reason"]?
|
|
end
|
|
end
|
|
|
|
if reason = info["reason"]?
|
|
if reason == "Video unavailable"
|
|
raise NotFoundException.new(reason.as_s || "")
|
|
else
|
|
raise InfoException.new(reason.as_s || "")
|
|
end
|
|
end
|
|
|
|
video = Video.new({
|
|
id: id,
|
|
info: info,
|
|
updated: Time.utc,
|
|
})
|
|
|
|
return video
|
|
end
|
|
|
|
def process_continuation(query, plid, id)
|
|
continuation = nil
|
|
if plid
|
|
if index = query["index"]?.try &.to_i?
|
|
continuation = index
|
|
else
|
|
continuation = id
|
|
end
|
|
continuation ||= 0
|
|
end
|
|
|
|
continuation
|
|
end
|
|
|
|
def build_thumbnails(id)
|
|
return {
|
|
{host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"},
|
|
{host: HOST_URL, height: 720, width: 1280, name: "maxresdefault", url: "maxresdefault"},
|
|
{host: HOST_URL, height: 480, width: 640, name: "sddefault", url: "sddefault"},
|
|
{host: HOST_URL, height: 360, width: 480, name: "high", url: "hqdefault"},
|
|
{host: HOST_URL, height: 180, width: 320, name: "medium", url: "mqdefault"},
|
|
{host: HOST_URL, height: 90, width: 120, name: "default", url: "default"},
|
|
{host: HOST_URL, height: 90, width: 120, name: "start", url: "1"},
|
|
{host: HOST_URL, height: 90, width: 120, name: "middle", url: "2"},
|
|
{host: HOST_URL, height: 90, width: 120, name: "end", url: "3"},
|
|
}
|
|
end
|