From f75a81c9eeb792c0b99075bc47a1243a23b8700b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Mon, 21 Feb 2022 10:53:20 +0100 Subject: [PATCH 1/9] Make configurable time between each RefreshChannelsJob --- config/config.example.yml | 8 ++++ docker-compose.yml | 2 + src/invidious/config.cr | 12 +++--- src/invidious/helpers/utils.cr | 44 +++++++++++++++------- src/invidious/jobs/refresh_channels_job.cr | 5 +-- 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 59cb486b..475f2703 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -314,6 +314,14 @@ https_only: false ## channel_threads: 1 +## +## Time between channel_refresh +## +## Accepted values: a valid time interval (hours:min:seconds) +## Default: 00:30:00 +## +channel_refresh_time: 00:30:00 + ## ## Forcefully dump and re-download the entire list of uploaded ## videos when crawling channel (during subscriptions update). diff --git a/docker-compose.yml b/docker-compose.yml index c76c314c..964bb702 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,9 @@ services: environment: # Adapted from ./config/config.yml INVIDIOUS_CONFIG: | + log_level: Info channel_threads: 1 + channel_refresh_time: 00:30:00 check_tables: true feed_threads: 1 db: diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 72e145da..150f8064 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -56,11 +56,13 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) + property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) + @[YAML::Field(converter: TimeSpanConverter)] + property channel_refresh_time : Time::Span = 30.minutes # Time between channel_refresh + property feed_threads : Int32 = 1 # Number of threads to use for updating feeds + property output : String = "STDOUT" # Log file path or STDOUT + property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) @[YAML::Field(converter: Preferences::URIConverter)] property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a58a21b1..2702c5e9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -18,23 +18,39 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def decode_length_seconds(string) - length_seconds = string.gsub(/[^0-9:]/, "") - return 0_i32 if length_seconds.empty? +module TimeSpanConverter + def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) + return yaml.scalar recode_length_seconds(value.total_seconds.to_i32) + end - length_seconds = length_seconds.split(":").map { |x| x.to_i? || 0 } - length_seconds = [0] * (3 - length_seconds.size) + length_seconds - - length_seconds = Time::Span.new( - hours: length_seconds[0], - minutes: length_seconds[1], - seconds: length_seconds[2] - ).total_seconds.to_i32 - - return length_seconds + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span + if node.is_a?(YAML::Nodes::Scalar) + return decode_time_span(node.value) + else + node.raise "Expected scalar, not #{node.class}" + end + end end -def recode_length_seconds(time) +def decode_time_span(string : String) : Time::Span + time_span = string.gsub(/[^0-9:]/, "") + return Time::Span.new(seconds: 0) if time_span.empty? + + time_span = time_span.split(":").map { |x| x.to_i? || 0 } + time_span = [0] * (3 - time_span.size) + time_span + + return Time::Span.new( + hours: time_span[0], + minutes: time_span[1], + seconds: time_span[2] + ) +end + +def decode_length_seconds(string : String) : Int32 + return decode_time_span(string).total_seconds.to_i32 +end + +def recode_length_seconds(time : Int32) : String if time <= 0 return "" else diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 55fb8154..3e04d1cd 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -58,9 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end end - # TODO: make this configurable - LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes") - sleep 30.minutes + LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_time}") + sleep CONFIG.channel_refresh_time Fiber.yield end end From 18197e7e3eca53176e71b1dfe82dd2de85df175c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Mon, 21 Feb 2022 11:13:24 +0100 Subject: [PATCH 2/9] Lint description of channel_refresh_time --- config/config.example.yml | 2 +- src/invidious/config.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 475f2703..5d4cea28 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -315,7 +315,7 @@ https_only: false channel_threads: 1 ## -## Time between channel_refresh +## Time between two jobs for crawling videos from channels ## ## Accepted values: a valid time interval (hours:min:seconds) ## Default: 00:30:00 diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 150f8064..9ffd2cd8 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -58,7 +58,7 @@ class Config property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) @[YAML::Field(converter: TimeSpanConverter)] - property channel_refresh_time : Time::Span = 30.minutes # Time between channel_refresh + property channel_refresh_time : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels property feed_threads : Int32 = 1 # Number of threads to use for updating feeds property output : String = "STDOUT" # Log file path or STDOUT property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr From dfab62ce48ff1d2227d77c2fc61f7a4ea8da1988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 00:46:26 +0100 Subject: [PATCH 3/9] Rename new property to channel_refresh_interval Follow indications : https://github.com/iv-org/invidious/pull/2915#discussion_r811373503 --- config/config.example.yml | 2 +- docker-compose.yml | 2 +- src/invidious/config.cr | 2 +- src/invidious/jobs/refresh_channels_job.cr | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 5d4cea28..9519f34a 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -320,7 +320,7 @@ channel_threads: 1 ## Accepted values: a valid time interval (hours:min:seconds) ## Default: 00:30:00 ## -channel_refresh_time: 00:30:00 +channel_refresh_interval: 00:30:00 ## ## Forcefully dump and re-download the entire list of uploaded diff --git a/docker-compose.yml b/docker-compose.yml index 964bb702..ab35a496 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: INVIDIOUS_CONFIG: | log_level: Info channel_threads: 1 - channel_refresh_time: 00:30:00 + channel_refresh_interval: 00:30:00 check_tables: true feed_threads: 1 db: diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 9ffd2cd8..fc24c1e7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -58,7 +58,7 @@ class Config property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) @[YAML::Field(converter: TimeSpanConverter)] - property channel_refresh_time : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels + property channel_refresh_interval : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels property feed_threads : Int32 = 1 # Number of threads to use for updating feeds property output : String = "STDOUT" # Log file path or STDOUT property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 3e04d1cd..92681408 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -58,8 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end end - LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_time}") - sleep CONFIG.channel_refresh_time + LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_interval}") + sleep CONFIG.channel_refresh_interval Fiber.yield end end From 5d2f2690e287d4fc1275dd9d3dc283e607117c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 00:59:55 +0100 Subject: [PATCH 4/9] Lint config properties Follow lint indications : https://github.com/iv-org/invidious/pull/2915#discussion_r811375584 --- src/invidious/config.cr | 81 +++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index fc24c1e7..158a05cc 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -56,22 +56,35 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) + # Number of threads to use for crawling videos from channels (for updating subscriptions) + property channel_threads : Int32 = 1 + # Time between two jobs for crawling videos from channels @[YAML::Field(converter: TimeSpanConverter)] - property channel_refresh_interval : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) + property channel_refresh_interval : Time::Span = 30.minutes + # Number of threads to use for updating feeds + property feed_threads : Int32 = 1 + # Log file path or STDOUT + property output : String = "STDOUT" + # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property log_level : LogLevel = LogLevel::Info + # Database configuration with separate parameters (username, hostname, etc) + property db : DBConfig? = nil + # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] - property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax - property decrypt_polling : Bool = true # Use polling to keep decryption function up to date - property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel - property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// - property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions - property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required - property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property database_url : URI = URI.parse("") + # Use polling to keep decryption function up to date + property decrypt_polling : Bool = true + # Used for crawling channels: threads should check all videos uploaded by a channel + property full_refresh : Bool = false + # Used to tell Invidious it is behind a proxy, so links to resources should be https:// + property https_only : Bool? + # HMAC signing key for CSRF tokens and verifying pubsub subscriptions + property hmac_key : String? + # Domain to be used for links to resources on the site where an absolute URL is required + property domain : String? + # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property use_pubsub_feeds : Bool | Int32 = false property popular_enabled : Bool = true property captcha_enabled : Bool = true property login_enabled : Bool = true @@ -80,28 +93,42 @@ class Config property admins : Array(String) = [] of String property external_port : Int32? = nil property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("") - property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs - property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. - property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards - property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. - property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely - property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + # For compliance with DMCA, disables download widget using list of video IDs + property dmca_content : Array(String) = [] of String + # Check table integrity, automatically try to add any missing columns, create tables, etc. + property check_tables : Bool = false + # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards + property cache_annotations : Bool = false + # Optional banner to be displayed along top of page for announcements, etc. + property banner : String? = nil + # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely + property hsts : Bool? = true + # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + property disable_proxy : Bool? | Array(String)? = false # URL to the modified source code to be easily AGPL compliant # Will display in the footer, next to the main source code link property modified_source_code_url : String? = nil + # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) @[YAML::Field(converter: Preferences::FamilyConverter)] - property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - property port : Int32 = 3000 # Port to listen for connections (overridden by command line argument) - property host_binding : String = "0.0.0.0" # Host to bind (overridden by command line argument) - property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = false # Use quic transport for youtube api + property force_resolve : Socket::Family = Socket::Family::UNSPEC + # Port to listen for connections (overridden by command line argument) + property port : Int32 = 3000 + # Host to bind (overridden by command line argument) + property host_binding : String = "0.0.0.0" + # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property pool_size : Int32 = 100 + # Use quic transport for youtube api + property use_quic : Bool = false + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] - property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format - property captcha_key : String? = nil # Key for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha + property cookies : HTTP::Cookies = HTTP::Cookies.new + # Key for Anti-Captcha + property captcha_key : String? = nil + # API URL for Anti-Captcha + property captcha_api_url : String = "https://api.anti-captcha.com" def disabled?(option) case disabled = CONFIG.disable_proxy From f109d812a1d8ea92a4ebdeb7ce03ad93bcbd9a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 01:34:19 +0100 Subject: [PATCH 5/9] Move TimeSpanConverter with another Converters Follow indications : https://github.com/iv-org/invidious/pull/2915#discussion_r811373953 --- src/invidious/config.cr | 2 +- src/invidious/helpers/utils.cr | 14 -------------- src/invidious/user/preferences.cr | 16 +++++++++++++++- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 158a05cc..cf705d21 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -59,7 +59,7 @@ class Config # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 # Time between two jobs for crawling videos from channels - @[YAML::Field(converter: TimeSpanConverter)] + @[YAML::Field(converter: Preferences::TimeSpanConverter)] property channel_refresh_interval : Time::Span = 30.minutes # Number of threads to use for updating feeds property feed_threads : Int32 = 1 diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 2702c5e9..22575c57 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -18,20 +18,6 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -module TimeSpanConverter - def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) - return yaml.scalar recode_length_seconds(value.total_seconds.to_i32) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span - if node.is_a?(YAML::Nodes::Scalar) - return decode_time_span(node.value) - else - node.raise "Expected scalar, not #{node.class}" - end - end -end - def decode_time_span(string : String) : Time::Span time_span = string.gsub(/[^0-9:]/, "") return Time::Span.new(seconds: 0) if time_span.empty? diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index bf7ea401..c01bdd82 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -256,4 +256,18 @@ struct Preferences cookies end end -end + + module TimeSpanConverter + def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) + return yaml.scalar recode_length_seconds(value.total_seconds.to_i32) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span + if node.is_a?(YAML::Nodes::Scalar) + return decode_time_span(node.value) + else + node.raise "Expected scalar, not #{node.class}" + end + end + end +end \ No newline at end of file From fd0ac3a6719b3d6280ce2e784ad370c8d10a6129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 01:35:35 +0100 Subject: [PATCH 6/9] Update management of channel_refresh_interval Follow indications: https://github.com/iv-org/invidious/pull/2915#discussion_r811373503 --- config/config.example.yml | 6 +++--- docker-compose.yml | 2 +- src/invidious/helpers/utils.cr | 18 ++++++++++++++++++ src/invidious/user/preferences.cr | 6 +++--- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 9519f34a..a6440e3d 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -317,10 +317,10 @@ channel_threads: 1 ## ## Time between two jobs for crawling videos from channels ## -## Accepted values: a valid time interval (hours:min:seconds) -## Default: 00:30:00 +## Accepted values: a valid time interval (like 1h30m or 90min) +## Default: 30m ## -channel_refresh_interval: 00:30:00 +channel_refresh_interval: 30min ## ## Forcefully dump and re-download the entire list of uploaded diff --git a/docker-compose.yml b/docker-compose.yml index ab35a496..4fdad921 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: INVIDIOUS_CONFIG: | log_level: Info channel_threads: 1 - channel_refresh_interval: 00:30:00 + channel_refresh_interval: 30m check_tables: true feed_threads: 1 db: diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 22575c57..53f64f4e 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -53,6 +53,24 @@ def recode_length_seconds(time : Int32) : String end end +def decode_interval(string : String) : Time::Span + rawMinutes = string.try &.to_i32? + + if !rawMinutes + hours = /(?\d+)h/.match(string).try &.["hours"].try &.to_i32 + hours ||= 0 + + minutes = /(?\d+)m(?!s)/.match(string).try &.["minutes"].try &.to_i32 + minutes ||= 0 + + time = Time::Span.new(hours: hours, minutes: minutes) + else + time = Time::Span.new(minutes: rawMinutes) + end + + return time +end + def decode_time(string) time = string.try &.to_f? diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index c01bdd82..9eeed53f 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -259,15 +259,15 @@ struct Preferences module TimeSpanConverter def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) - return yaml.scalar recode_length_seconds(value.total_seconds.to_i32) + return yaml.scalar value.total_minutes.to_i32 end def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span if node.is_a?(YAML::Nodes::Scalar) - return decode_time_span(node.value) + return decode_interval(node.value) else node.raise "Expected scalar, not #{node.class}" end end end -end \ No newline at end of file +end From afa3eff313428ad5cb9783a12ce4884d207c11c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 08:07:50 +0100 Subject: [PATCH 7/9] Remove useless config inside docker-compose Follow lint indications : https://github.com/iv-org/invidious/pull/2915#discussion_r811501709 --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4fdad921..c76c314c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,9 +23,7 @@ services: environment: # Adapted from ./config/config.yml INVIDIOUS_CONFIG: | - log_level: Info channel_threads: 1 - channel_refresh_interval: 30m check_tables: true feed_threads: 1 db: From 555bb711c9bb16f8a78a8b71146367438b81918e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 08:17:50 +0100 Subject: [PATCH 8/9] Removal of changes to methods now unrelated to the issue Unrelated to the issue since the change in management of channel_refresh_interval Cf this remark : https://github.com/iv-org/invidious/pull/2915#discussion_r811373503 --- src/invidious/helpers/utils.cr | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 53f64f4e..c1dc17db 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -18,25 +18,23 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def decode_time_span(string : String) : Time::Span - time_span = string.gsub(/[^0-9:]/, "") - return Time::Span.new(seconds: 0) if time_span.empty? +def decode_length_seconds(string) + length_seconds = string.gsub(/[^0-9:]/, "") + return 0_i32 if length_seconds.empty? - time_span = time_span.split(":").map { |x| x.to_i? || 0 } - time_span = [0] * (3 - time_span.size) + time_span + length_seconds = length_seconds.split(":").map { |x| x.to_i? || 0 } + length_seconds = [0] * (3 - length_seconds.size) + length_seconds - return Time::Span.new( - hours: time_span[0], - minutes: time_span[1], - seconds: time_span[2] - ) + length_seconds = Time::Span.new( + hours: length_seconds[0], + minutes: length_seconds[1], + seconds: length_seconds[2] + ).total_seconds.to_i32 + + return length_seconds end -def decode_length_seconds(string : String) : Int32 - return decode_time_span(string).total_seconds.to_i32 -end - -def recode_length_seconds(time : Int32) : String +def recode_length_seconds(time) if time <= 0 return "" else From e60a1836fe077fa00fbfc8e8f9e4401b7151192c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Tue, 22 Feb 2022 23:19:59 +0100 Subject: [PATCH 9/9] Lint config.example.yml and config.cr Follow lint indications : - https://github.com/iv-org/invidious/pull/2915#discussion_r812396203 - https://github.com/iv-org/invidious/pull/2915#discussion_r812396807 --- config/config.example.yml | 7 ++++--- src/invidious/config.cr | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index a6440e3d..866e8944 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -315,12 +315,13 @@ https_only: false channel_threads: 1 ## -## Time between two jobs for crawling videos from channels +## Time interval between two executions of the job that crawls +## channel videos (subscriptions update). ## -## Accepted values: a valid time interval (like 1h30m or 90min) +## Accepted values: a valid time interval (like 1h30m or 90m) ## Default: 30m ## -channel_refresh_interval: 30min +#channel_refresh_interval: 30m ## ## Forcefully dump and re-download the entire list of uploaded diff --git a/src/invidious/config.cr b/src/invidious/config.cr index cf705d21..bebb9ae5 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -58,7 +58,7 @@ class Config # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 - # Time between two jobs for crawling videos from channels + # Time interval between two executions of the job that crawls channel videos (subscriptions update). @[YAML::Field(converter: Preferences::TimeSpanConverter)] property channel_refresh_interval : Time::Span = 30.minutes # Number of threads to use for updating feeds